github/coffeescript
1
1
Fork 0
mirror of https://github.com/jashkenas/coffeescript.git synced 2026-05-03 03:00:14 -04:00
Code Issues Packages Projects Releases Wiki Activity

Basic comments AST; PassthroughLiteral AST (#5220)

Browse Source 
* passing existing tests

* comments ast

* fixes from code review
This commit is contained in:
Julian Rosse
2019-07-21 21:05:05 -04:00
committed by Geoffrey Booth
parent f33d4dd4f1
commit 1f22c16bee
11 changed files with 808 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
lib/coffeescript/coffeescript.js
View File

@@ -146,6 +146,7 @@
// which might’ve gotten misaligned from the original source due to the
// `clean` function in the lexer).
if (options.ast) {
nodes.allCommentTokens = helpers.extractAllCommentTokens(tokens);
sourceCodeNumberOfLines = (code.match(/\r?\n/g) || '').length + 1;
sourceCodeLastLine = /.*$/.exec(code)[0];
ast = nodes.ast(options);

37
lib/coffeescript/helpers.js
View File

@@ -165,6 +165,32 @@
}
};
// Build a list of all comments attached to tokens.
exports.extractAllCommentTokens = function(tokens) {
var allCommentsObj, comment, commentKey, i, j, k, key, len1, len2, len3, ref1, results, sortedKeys, token;
allCommentsObj = {};
for (i = 0, len1 = tokens.length; i < len1; i++) {
token = tokens[i];
if (token.comments) {
ref1 = token.comments;
for (j = 0, len2 = ref1.length; j < len2; j++) {
comment = ref1[j];
commentKey = comment.locationData.range[0];
allCommentsObj[commentKey] = comment;
}
}
}
sortedKeys = Object.keys(allCommentsObj).sort(function(a, b) {
return a - b;
});
results = [];
for (k = 0, len3 = sortedKeys.length; k < len3; k++) {
key = sortedKeys[k];
results.push(allCommentsObj[key]);
}
return results;
};
// Get a lookup hash for a token based on its location data.
// Multiple tokens might have the same location hash, but using exclusive
// location data distinguishes e.g. zero-length generated tokens from
@@ -175,12 +201,11 @@
// Build a dictionary of extra token properties organized by tokens’ locations
// used as lookup hashes.
buildTokenDataDictionary = function(parserState) {
var base, i, len1, ref1, token, tokenData, tokenHash;
exports.buildTokenDataDictionary = buildTokenDataDictionary = function(tokens) {
var base, i, len1, token, tokenData, tokenHash;
tokenData = {};
ref1 = parserState.parser.tokens;
for (i = 0, len1 = ref1.length; i < len1; i++) {
token = ref1[i];
for (i = 0, len1 = tokens.length; i < len1; i++) {
token = tokens[i];
if (!token.comments) {
continue;
}
@@ -218,7 +243,7 @@
// Add comments, building the dictionary of token data if it hasn’t been
// built yet.
if (parserState.tokenData == null) {
parserState.tokenData = buildTokenDataDictionary(parserState);
parserState.tokenData = buildTokenDataDictionary(parserState.parser.tokens);
}
if (obj.locationData != null) {
objHash = buildLocationHash(obj.locationData);

130
lib/coffeescript/lexer.js
View File

@@ -112,7 +112,7 @@
// Tokenizers
// ----------
// Matches identifying literals: variables, keywords, method names, etc.
// Matches identifying literals: variables, keywords, method names, etc.
// Check to ensure that JavaScript reserved words aren’t being used as
// identifiers. Because CoffeeScript reserves a handful of keywords that are
// allowed in JavaScript, we’re careful not to tag them as keywords when
@@ -399,71 +399,113 @@
// stream and saved for later, to be reinserted into the output after
// everything has been parsed and the JavaScript code generated.
commentToken(chunk = this.chunk) {
var comment, commentAttachments, content, contents, here, i, match, matchIllegal, newLine, placeholderToken, prev;
var commentAttachment, commentAttachments, commentWithSurroundingWhitespace, content, contents, hasSeenFirstCommentLine, hereComment, hereLeadingWhitespace, hereTrailingWhitespace, i, leadingNewline, leadingNewlineOffset, leadingNewlines, leadingWhitespace, length, lineComment, match, matchIllegal, nonInitial, offsetInChunk, placeholderToken, precededByBlankLine, precedingNonCommentLines, prev;
if (!(match = chunk.match(COMMENT))) {
return 0;
}
[comment, here] = match;
[commentWithSurroundingWhitespace, hereLeadingWhitespace, hereComment, hereTrailingWhitespace, lineComment] = match;
contents = null;
// Does this comment follow code on the same line?
newLine = /^\s*\n+\s*#/.test(comment);
if (here) {
matchIllegal = HERECOMMENT_ILLEGAL.exec(comment);
leadingNewline = /^\s*\n+\s*#/.test(commentWithSurroundingWhitespace);
if (hereComment) {
matchIllegal = HERECOMMENT_ILLEGAL.exec(hereComment);
if (matchIllegal) {
this.error(`block comments cannot contain ${matchIllegal[0]}`, {
offset: matchIllegal.index,
offset: '###'.length + matchIllegal.index,
length: matchIllegal[0].length
});
}
// Parse indentation or outdentation as if this block comment didn’t exist.
chunk = chunk.replace(`###${here}###`, '');
chunk = chunk.replace(`###${hereComment}###`, '');
// Remove leading newlines, like `Rewriter::removeLeadingNewlines`, to
// avoid the creation of unwanted `TERMINATOR` tokens.
chunk = chunk.replace(/^\n+/, '');
this.lineToken(chunk);
this.lineToken({chunk});
// Pull out the ###-style comment’s content, and format it.
content = here;
if (indexOf.call(content, '\n') >= 0) {
content = content.replace(RegExp(`\\n${repeat(' ', this.indent)}`, "g"), '\n');
}
contents = [content];
content = hereComment;
contents = [
{
content,
length: commentWithSurroundingWhitespace.length - hereLeadingWhitespace.length - hereTrailingWhitespace.length,
leadingWhitespace: hereLeadingWhitespace
}
];
} else {
// The `COMMENT` regex captures successive line comments as one token.
// Remove any leading newlines before the first comment, but preserve
// blank lines between line comments.
content = comment.replace(/^(\n*)/, '');
content = content.replace(/^([ |\t]*)#/gm, '');
contents = content.split('\n');
leadingNewlines = '';
content = lineComment.replace(/^(\n*)/, function(leading) {
leadingNewlines = leading;
return '';
});
precedingNonCommentLines = '';
hasSeenFirstCommentLine = false;
contents = content.split('\n').map(function(line, index) {
var comment, leadingWhitespace;
if (!(line.indexOf('#') > -1)) {
precedingNonCommentLines += `\n${line}`;
return;
}
leadingWhitespace = '';
content = line.replace(/^([ |\t]*)#/, function(_, whitespace) {
leadingWhitespace = whitespace;
return '';
});
comment = {
content,
length: '#'.length + content.length,
leadingWhitespace: `${!hasSeenFirstCommentLine ? leadingNewlines : ''}${precedingNonCommentLines}${leadingWhitespace}`,
precededByBlankLine: !!precedingNonCommentLines
};
hasSeenFirstCommentLine = true;
precedingNonCommentLines = '';
return comment;
}).filter(function(comment) {
return comment;
});
}
offsetInChunk = 0;
commentAttachments = (function() {
var j, len, results;
results = [];
for (i = j = 0, len = contents.length; j < len; i = ++j) {
content = contents[i];
results.push({
content: content,
here: here != null,
newLine: newLine || i !== 0 // Line comments after the first one start new lines, by definition.
});
({content, length, leadingWhitespace, precededByBlankLine} = contents[i]);
nonInitial = i !== 0;
leadingNewlineOffset = nonInitial ? 1 : 0;
offsetInChunk += leadingNewlineOffset + leadingWhitespace.length;
commentAttachment = {
content,
here: hereComment != null,
newLine: leadingNewline || nonInitial, // Line comments after the first one start new lines, by definition.
locationData: this.makeLocationData({offsetInChunk, length}),
precededByBlankLine
};
offsetInChunk += length;
results.push(commentAttachment);
}
return results;
})();
}).call(this);
prev = this.prev();
if (!prev) {
// If there’s no previous token, create a placeholder token to attach
// this comment to; and follow with a newline.
commentAttachments[0].newLine = true;
this.lineToken(this.chunk.slice(comment.length));
this.lineToken({
chunk: this.chunk.slice(commentWithSurroundingWhitespace.length),
offset: commentWithSurroundingWhitespace.length // Set the indent.
});
placeholderToken = this.makeToken('JS', '', {
offset: commentWithSurroundingWhitespace.length,
generated: true
});
placeholderToken.comments = commentAttachments;
this.tokens.push(placeholderToken);
this.newlineToken(0);
this.newlineToken(commentWithSurroundingWhitespace.length);
} else {
attachCommentsToNode(commentAttachments, prev);
}
return comment.length;
return commentWithSurroundingWhitespace.length;
}
// Matches JavaScript interpolated directly into the source via backticks.
@@ -605,7 +647,7 @@
// Keeps track of the level of indentation, because a single outdent token
// can close multiple indents, so we need to know how far in we happen to be.
lineToken(chunk = this.chunk) {
lineToken({chunk = this.chunk, offset = 0} = {}) {
var backslash, diff, indent, match, minLiteralLength, newIndentLiteral, noNewlines, prev, size;
if (!(match = MULTI_DENT.exec(chunk))) {
return 0;
@@ -642,7 +684,7 @@
if (noNewlines) {
this.suppressNewlines();
} else {
this.newlineToken(0);
this.newlineToken(offset);
}
return indent.length;
}
@@ -661,7 +703,7 @@
}
diff = size - this.indent + this.outdebt;
this.token('INDENT', diff, {
offset: indent.length - size,
offset: offset + indent.length - size,
length: size
});
this.indents.push(diff);
@@ -673,18 +715,23 @@
this.indentLiteral = newIndentLiteral;
} else if (size < this.baseIndent) {
this.error('missing indentation', {
offset: indent.length
offset: offset + indent.length
});
} else {
this.indebt = 0;
this.outdentToken(this.indent - size, noNewlines, indent.length);
this.outdentToken({
moveOut: this.indent - size,
noNewlines,
outdentLength: indent.length,
offset
});
}
return indent.length;
}
// Record an outdent token or multiple tokens, if we happen to be moving back
// inwards past several recorded indents. Sets new @indent value.
outdentToken(moveOut, noNewlines, outdentLength) {
outdentToken({moveOut, noNewlines, outdentLength = 0, offset = 0}) {
var decreasedIndent, dent, lastIndent, ref;
decreasedIndent = this.indent - moveOut;
while (moveOut > 0) {
@@ -715,7 +762,7 @@
this.suppressSemicolons();
if (!(this.tag() === 'TERMINATOR' || noNewlines)) {
this.token('TERMINATOR', '\n', {
offset: outdentLength,
offset: offset + outdentLength,
length: 0
});
}
@@ -1067,7 +1114,7 @@
// Token Manipulators
// ------------------
// A source of ambiguity in our grammar used to be parameter lists in function
// A source of ambiguity in our grammar used to be parameter lists in function
// definitions versus argument lists in function calls. Walk backwards, tagging
// parameters specially in order to make things easier for the parser.
tagParameters() {
@@ -1115,7 +1162,9 @@
// Close up all remaining open blocks at the end of the file.
closeIndentation() {
return this.outdentToken(this.indent);
return this.outdentToken({
moveOut: this.indent
});
}
// Match the contents of a delimited token and expand variables and expressions
@@ -1345,7 +1394,10 @@
// el.hide())
ref1 = this.indents, [lastIndent] = slice.call(ref1, -1);
this.outdentToken(lastIndent, true);
this.outdentToken({
moveOut: lastIndent,
noNewlines: true
});
return this.pair(tag);
}
return this.ends.pop();
@@ -1354,7 +1406,7 @@
// Helpers
// -------
// Returns the line and column number from an offset into the current chunk.
// Returns the line and column number from an offset into the current chunk.
// `offset` is a number of characters into `@chunk`.
getLineAndColumnFromChunk(offset) {
@@ -1618,7 +1670,7 @@
WHITESPACE = /^[^\n\S]+/;
COMMENT = /^\s*###([^#][\s\S]*?)(?:###[^\n\S]*|###$)|^(?:\s*#(?!##[^#]).*)+/;
COMMENT = /^(\s*)###([^#][\s\S]*?)(?:###([^\n\S]*)|###$)|^((?:\s*#(?!##[^#]).*)+)/;
CODE = /^[-=]>/;

122
lib/coffeescript/nodes.js
View File

@@ -703,6 +703,33 @@
return results;
}
commentsAst() {
var comment, commentToken, j, len1, ref1, results;
if (this.allComments == null) {
this.allComments = (function() {
var j, len1, ref1, ref2, results;
ref2 = (ref1 = this.allCommentTokens) != null ? ref1 : [];
results = [];
for (j = 0, len1 = ref2.length; j < len1; j++) {
commentToken = ref2[j];
if (commentToken.here) {
results.push(new HereComment(commentToken));
} else {
results.push(new LineComment(commentToken));
}
}
return results;
}).call(this);
}
ref1 = this.allComments;
results = [];
for (j = 0, len1 = ref1.length; j < len1; j++) {
comment = ref1[j];
results.push(comment.ast());
}
return results;
}
ast(o) {
o.level = LEVEL_TOP;
this.initializeScope(o);
@@ -717,7 +744,7 @@
this.body.isRootBlock = true;
return {
program: Object.assign(this.body.ast(o), this.astLocationData()),
comments: []
comments: this.commentsAst()
};
}
@@ -1133,7 +1160,10 @@
for (j = 0, len1 = ref1.length; j < len1; j++) {
expression = ref1[j];
expressionAst = expression.ast(o);
if (expression instanceof Directive) {
// Ignore generated PassthroughLiteral
if (expressionAst == null) {
continue;
} else if (expression instanceof Directive) {
directives.push(expressionAst);
// If an expression is a statement, it can be added to the body as is.
} else if (expression.isStatementAst(o)) {
@@ -1158,7 +1188,7 @@
// or `script`, and so if Node figures out a way to do so for `.js` files
// then CoffeeScript can copy Node’s algorithm.
// sourceType: 'module'
// sourceType: 'module'
return {body, directives};
}
@@ -1495,6 +1525,20 @@
});
}
ast(o, level) {
if (this.generated) {
return null;
}
return super.ast(o, level);
}
astProperties() {
return {
value: this.value,
here: !!this.here
};
}
};
exports.IdentifierLiteral = IdentifierLiteral = (function() {
@@ -2184,33 +2228,37 @@
constructor({
content: content1,
newLine,
unshift
unshift,
locationData: locationData1
}) {
super();
this.content = content1;
this.newLine = newLine;
this.unshift = unshift;
this.locationData = locationData1;
}
compileNode(o) {
var fragment, hasLeadingMarks, j, largestIndent, leadingWhitespace, len1, line, multiline, ref1;
var fragment, hasLeadingMarks, indent, j, leadingWhitespace, len1, line, multiline, ref1;
multiline = indexOf.call(this.content, '\n') >= 0;
hasLeadingMarks = /\n\s*[#|\*]/.test(this.content);
if (hasLeadingMarks) {
this.content = this.content.replace(/^([ \t]*)#(?=\s)/gm, ' *');
}
// Unindent multiline comments. They will be reindented later.
if (multiline) {
largestIndent = '';
indent = null;
ref1 = this.content.split('\n');
for (j = 0, len1 = ref1.length; j < len1; j++) {
line = ref1[j];
leadingWhitespace = /^\s*/.exec(line)[0];
if (leadingWhitespace.length > largestIndent.length) {
largestIndent = leadingWhitespace;
if (!indent || leadingWhitespace.length < indent.length) {
indent = leadingWhitespace;
}
}
this.content = this.content.replace(RegExp(`^(${leadingWhitespace})`, "gm"), '');
if (indent) {
this.content = this.content.replace(RegExp(`\\n${indent}`, "g"), '\n');
}
}
hasLeadingMarks = /\n\s*[#|\*]/.test(this.content);
if (hasLeadingMarks) {
this.content = this.content.replace(/^([ \t]*)#(?=\s)/gm, ' *');
}
this.content = `/*${this.content}${hasLeadingMarks ? ' ' : ''}*/`;
fragment = this.makeCode(this.content);
@@ -2222,6 +2270,16 @@
return fragment;
}
astType() {
return 'CommentBlock';
}
astProperties() {
return {
value: this.content
};
}
};
//### LineComment
@@ -2231,17 +2289,21 @@
constructor({
content: content1,
newLine,
unshift
unshift,
locationData: locationData1,
precededByBlankLine
}) {
super();
this.content = content1;
this.newLine = newLine;
this.unshift = unshift;
this.locationData = locationData1;
this.precededByBlankLine = precededByBlankLine;
}
compileNode(o) {
var fragment;
fragment = this.makeCode(/^\s*$/.test(this.content) ? '' : `//${this.content}`);
fragment = this.makeCode(/^\s*$/.test(this.content) ? '' : `${this.precededByBlankLine ? `\n${o.indent}` : ''}//${this.content}`);
fragment.newLine = this.newLine;
fragment.unshift = this.unshift;
fragment.trail = !this.newLine && !this.unshift;
@@ -2250,6 +2312,16 @@
return fragment;
}
astType() {
return 'CommentLine';
}
astProperties() {
return {
value: this.content
};
}
};
//### JSX
@@ -7092,17 +7164,23 @@
attachCommentsToNode(salvagedComments, node);
}
if ((unwrapped = (ref1 = node.expression) != null ? ref1.unwrapAll() : void 0) instanceof PassthroughLiteral && unwrapped.generated && !this.jsx) {
commentPlaceholder = new StringLiteral('').withLocationDataFrom(node);
commentPlaceholder.comments = unwrapped.comments;
if (node.comments) {
(commentPlaceholder.comments != null ? commentPlaceholder.comments : commentPlaceholder.comments = []).push(...node.comments);
if (o.compiling) {
commentPlaceholder = new StringLiteral('').withLocationDataFrom(node);
commentPlaceholder.comments = unwrapped.comments;
if (node.comments) {
(commentPlaceholder.comments != null ? commentPlaceholder.comments : commentPlaceholder.comments = []).push(...node.comments);
}
elements.push(new Value(commentPlaceholder));
} else {
elements.push(null);
}
elements.push(new Value(commentPlaceholder));
} else if (node.expression || includeInterpolationWrappers) {
if (node.comments) {
((ref2 = node.expression) != null ? ref2.comments != null ? ref2.comments : ref2.comments = [] : void 0).push(...node.comments);
}
elements.push(includeInterpolationWrappers ? node : node.expression);
} else if (!o.compiling) {
elements.push(null);
}
return false;
} else if (node.comments) {
@@ -7186,7 +7264,7 @@
}
astProperties(o) {
var element, elements, expressions, index, j, last, len1, quasis;
var element, elements, expressions, index, j, last, len1, quasis, ref1;
elements = this.extractElements(o);
[last] = slice1.call(elements, -1);
quasis = [];
@@ -7198,7 +7276,7 @@
tail: element === last
}).withLocationDataFrom(element).ast(o));
} else {
expressions.push(element.unwrap().ast(o));
expressions.push((ref1 = element != null ? element.unwrap().ast(o) : void 0) != null ? ref1 : null);
}
}
return {expressions, quasis, quote: this.quote};

6
lib/coffeescript/sourcemap.js
View File

@@ -53,10 +53,10 @@
// SourceMap
// ---------
// Maps locations in a single generated JavaScript file back to locations in
// Maps locations in a single generated JavaScript file back to locations in
// the original CoffeeScript source file.
// This is intentionally agnostic towards how a source map might be represented on
// This is intentionally agnostic towards how a source map might be represented on
// disk. Once the compiler is ready to produce a "v3"-style source map, we can walk
// through the arrays of line and column buffer to produce it.
class SourceMap {
@@ -88,7 +88,7 @@
// V3 SourceMap Generation
// -----------------------
// Builds up a V3 source map, returning the generated JSON as a string.
// Builds up a V3 source map, returning the generated JSON as a string.
// `options.sourceRoot` may be used to specify the sourceRoot written to the source
// map. Also, `options.sourceFiles` and `options.generatedFile` may be passed to
// set "sources" and "file", respectively.

1
src/coffeescript.coffee
View File

@@ -113,6 +113,7 @@ exports.compile = compile = withPrettyErrors (code, options = {}) ->
# which might’ve gotten misaligned from the original source due to the
# `clean` function in the lexer).
if options.ast
nodes.allCommentTokens = helpers.extractAllCommentTokens tokens
sourceCodeNumberOfLines = (code.match(/\r?\n/g) or '').length + 1
sourceCodeLastLine = /.*$/.exec(code)[0] # `.*` matches all but line break characters.
ast = nodes.ast options

17
src/helpers.coffee
View File

@@ -114,6 +114,17 @@ buildLocationData = (first, last) ->
last.range[1]
]
# Build a list of all comments attached to tokens.
exports.extractAllCommentTokens = (tokens) ->
allCommentsObj = {}
for token in tokens when token.comments
for comment in token.comments
commentKey = comment.locationData.range[0]
allCommentsObj[commentKey] = comment
sortedKeys = Object.keys(allCommentsObj).sort (a, b) -> a - b
for key in sortedKeys
allCommentsObj[key]
# Get a lookup hash for a token based on its location data.
# Multiple tokens might have the same location hash, but using exclusive
# location data distinguishes e.g. zero-length generated tokens from
@@ -123,9 +134,9 @@ buildLocationHash = (loc) ->
# Build a dictionary of extra token properties organized by tokens’ locations
# used as lookup hashes.
buildTokenDataDictionary = (parserState) ->
exports.buildTokenDataDictionary = buildTokenDataDictionary = (tokens) ->
tokenData = {}
for token in parserState.parser.tokens when token.comments
for token in tokens when token.comments
tokenHash = buildLocationHash token[2]
# Multiple tokens might have the same location hash, such as the generated
# `JS` tokens added at the start or end of the token stream to hold
@@ -153,7 +164,7 @@ exports.addDataToNode = (parserState, firstLocationData, firstValue, lastLocatio
# Add comments, building the dictionary of token data if it hasn’t been
# built yet.
parserState.tokenData ?= buildTokenDataDictionary parserState
parserState.tokenData ?= buildTokenDataDictionary parserState.parser.tokens
if obj.locationData?
objHash = buildLocationHash obj.locationData
if parserState.tokenData[objHash]?.comments?

99
src/lexer.coffee
View File

@@ -314,55 +314,90 @@ exports.Lexer = class Lexer
# everything has been parsed and the JavaScript code generated.
commentToken: (chunk = @chunk) ->
return 0 unless match = chunk.match COMMENT
[comment, here] = match
[commentWithSurroundingWhitespace, hereLeadingWhitespace, hereComment, hereTrailingWhitespace, lineComment] = match
contents = null
# Does this comment follow code on the same line?
newLine = /^\s*\n+\s*#/.test comment
if here
matchIllegal = HERECOMMENT_ILLEGAL.exec comment
leadingNewline = /^\s*\n+\s*#/.test commentWithSurroundingWhitespace
if hereComment
matchIllegal = HERECOMMENT_ILLEGAL.exec hereComment
if matchIllegal
@error "block comments cannot contain #{matchIllegal[0]}",
offset: matchIllegal.index, length: matchIllegal[0].length
offset: '###'.length + matchIllegal.index, length: matchIllegal[0].length
# Parse indentation or outdentation as if this block comment didn’t exist.
chunk = chunk.replace "####{here}###", ''
chunk = chunk.replace "####{hereComment}###", ''
# Remove leading newlines, like `Rewriter::removeLeadingNewlines`, to
# avoid the creation of unwanted `TERMINATOR` tokens.
chunk = chunk.replace /^\n+/, ''
@lineToken chunk
@lineToken {chunk}
# Pull out the ###-style comment’s content, and format it.
content = here
if '\n' in content
content = content.replace /// \n #{repeat ' ', @indent} ///g, '\n'
contents = [content]
content = hereComment
contents = [{
content
length: commentWithSurroundingWhitespace.length - hereLeadingWhitespace.length - hereTrailingWhitespace.length
leadingWhitespace: hereLeadingWhitespace
}]
else
# The `COMMENT` regex captures successive line comments as one token.
# Remove any leading newlines before the first comment, but preserve
# blank lines between line comments.
content = comment.replace /^(\n*)/, ''
content = content.replace /^([ |\t]*)#/gm, ''
contents = content.split '\n'
leadingNewlines = ''
content = lineComment.replace /^(\n*)/, (leading) ->
leadingNewlines = leading
''
precedingNonCommentLines = ''
hasSeenFirstCommentLine = no
contents =
content.split '\n'
.map (line, index) ->
unless line.indexOf('#') > -1
precedingNonCommentLines += "\n#{line}"
return
leadingWhitespace = ''
content = line.replace /^([ |\t]*)#/, (_, whitespace) ->
leadingWhitespace = whitespace
''
comment = {
content
length: '#'.length + content.length
leadingWhitespace: "#{unless hasSeenFirstCommentLine then leadingNewlines else ''}#{precedingNonCommentLines}#{leadingWhitespace}"
precededByBlankLine: !!precedingNonCommentLines
}
hasSeenFirstCommentLine = yes
precedingNonCommentLines = ''
comment
.filter (comment) -> comment
commentAttachments = for content, i in contents
content: content
here: here?
newLine: newLine or i isnt 0 # Line comments after the first one start new lines, by definition.
offsetInChunk = 0
commentAttachments = for {content, length, leadingWhitespace, precededByBlankLine}, i in contents
nonInitial = i isnt 0
leadingNewlineOffset = if nonInitial then 1 else 0
offsetInChunk += leadingNewlineOffset + leadingWhitespace.length
commentAttachment = {
content
here: hereComment?
newLine: leadingNewline or nonInitial # Line comments after the first one start new lines, by definition.
locationData: @makeLocationData {offsetInChunk, length}
precededByBlankLine
}
offsetInChunk += length
commentAttachment
prev = @prev()
unless prev
# If there’s no previous token, create a placeholder token to attach
# this comment to; and follow with a newline.
commentAttachments[0].newLine = yes
@lineToken @chunk[comment.length..] # Set the indent.
placeholderToken = @makeToken 'JS', '', generated: yes
@lineToken chunk: @chunk[commentWithSurroundingWhitespace.length..], offset: commentWithSurroundingWhitespace.length # Set the indent.
placeholderToken = @makeToken 'JS', '', offset: commentWithSurroundingWhitespace.length, generated: yes
placeholderToken.comments = commentAttachments
@tokens.push placeholderToken
@newlineToken 0
@newlineToken commentWithSurroundingWhitespace.length
else
attachCommentsToNode commentAttachments, prev
comment.length
commentWithSurroundingWhitespace.length
# Matches JavaScript interpolated directly into the source via backticks.
jsToken: ->
@@ -436,7 +471,7 @@ exports.Lexer = class Lexer
#
# Keeps track of the level of indentation, because a single outdent token
# can close multiple indents, so we need to know how far in we happen to be.
lineToken: (chunk = @chunk) ->
lineToken: ({chunk = @chunk, offset = 0} = {}) ->
return 0 unless match = MULTI_DENT.exec chunk
indent = match[0]
@@ -460,7 +495,7 @@ exports.Lexer = class Lexer
return indent.length
if size - @indebt is @indent
if noNewlines then @suppressNewlines() else @newlineToken 0
if noNewlines then @suppressNewlines() else @newlineToken offset
return indent.length
if size > @indent
@@ -473,22 +508,22 @@ exports.Lexer = class Lexer
@indentLiteral = newIndentLiteral
return indent.length
diff = size - @indent + @outdebt
@token 'INDENT', diff, offset: indent.length - size, length: size
@token 'INDENT', diff, offset: offset + indent.length - size, length: size
@indents.push diff
@ends.push {tag: 'OUTDENT'}
@outdebt = @indebt = 0
@indent = size
@indentLiteral = newIndentLiteral
else if size < @baseIndent
@error 'missing indentation', offset: indent.length
@error 'missing indentation', offset: offset + indent.length
else
@indebt = 0
@outdentToken @indent - size, noNewlines, indent.length
@outdentToken {moveOut: @indent - size, noNewlines, outdentLength: indent.length, offset}
indent.length
# Record an outdent token or multiple tokens, if we happen to be moving back
# inwards past several recorded indents. Sets new @indent value.
outdentToken: (moveOut, noNewlines, outdentLength) ->
outdentToken: ({moveOut, noNewlines, outdentLength = 0, offset = 0}) ->
decreasedIndent = @indent - moveOut
while moveOut > 0
lastIndent = @indents[@indents.length - 1]
@@ -510,7 +545,7 @@ exports.Lexer = class Lexer
@outdebt -= moveOut if dent
@suppressSemicolons()
@token 'TERMINATOR', '\n', offset: outdentLength, length: 0 unless @tag() is 'TERMINATOR' or noNewlines
@token 'TERMINATOR', '\n', offset: offset + outdentLength, length: 0 unless @tag() is 'TERMINATOR' or noNewlines
@indent = decreasedIndent
@indentLiteral = @indentLiteral[...decreasedIndent]
this
@@ -766,7 +801,7 @@ exports.Lexer = class Lexer
# Close up all remaining open blocks at the end of the file.
closeIndentation: ->
@outdentToken @indent
@outdentToken moveOut: @indent
# Match the contents of a delimited token and expand variables and expressions
# inside it using Ruby-like notation for substitution of arbitrary
@@ -938,7 +973,7 @@ exports.Lexer = class Lexer
# el.hide())
#
[..., lastIndent] = @indents
@outdentToken lastIndent, true
@outdentToken moveOut: lastIndent, noNewlines: true
return @pair tag
@ends.pop()
@@ -1193,7 +1228,7 @@ OPERATOR = /// ^ (
WHITESPACE = /^[^\n\S]+/
COMMENT = /^\s*###([^#][\s\S]*?)(?:###[^\n\S]*|###$)|^(?:\s*#(?!##[^#]).*)+/
COMMENT = /^(\s*)###([^#][\s\S]*?)(?:###([^\n\S]*)|###$)|^((?:\s*#(?!##[^#]).*)+)/
CODE = /^[-=]>/

72
src/nodes.coffee
View File

@@ -500,6 +500,15 @@ exports.Root = class Root extends Base
# end up being declared on the root block.
o.scope.parameter name for name in o.locals or []
commentsAst: ->
@allComments ?=
for commentToken in (@allCommentTokens ? [])
if commentToken.here
new HereComment commentToken
else
new LineComment commentToken
comment.ast() for comment in @allComments
ast: (o) ->
o.level = LEVEL_TOP
@initializeScope o
@@ -511,7 +520,7 @@ exports.Root = class Root extends Base
@body.isRootBlock = yes
return
program: Object.assign @body.ast(o), @astLocationData()
comments: []
comments: @commentsAst()
#### Block
@@ -806,7 +815,10 @@ exports.Block = class Block extends Base
body = []
for expression in @expressions
expressionAst = expression.ast o
if expression instanceof Directive
# Ignore generated PassthroughLiteral
if not expressionAst?
continue
else if expression instanceof Directive
directives.push expressionAst
# If an expression is a statement, it can be added to the body as is.
else if expression.isStatementAst o
@@ -1060,6 +1072,16 @@ exports.PassthroughLiteral = class PassthroughLiteral extends Literal
# By reducing it to its latter half, we turn '\`' to '`', '\\\`' to '\`', etc.
string[-Math.ceil(string.length / 2)..]
ast: (o, level) ->
return null if @generated
super o, level
astProperties: ->
return {
@value
here: !!@here
}
exports.IdentifierLiteral = class IdentifierLiteral extends Literal
isAssignable: YES
@@ -1483,22 +1505,23 @@ exports.MetaProperty = class MetaProperty extends Base
# Comment delimited by `###` (becoming `/* */`).
exports.HereComment = class HereComment extends Base
constructor: ({ @content, @newLine, @unshift }) ->
constructor: ({ @content, @newLine, @unshift, @locationData }) ->
super()
compileNode: (o) ->
multiline = '\n' in @content
hasLeadingMarks = /\n\s*[#|\*]/.test @content
@content = @content.replace /^([ \t]*)#(?=\s)/gm, ' *' if hasLeadingMarks
# Unindent multiline comments. They will be reindented later.
if multiline
largestIndent = ''
indent = null
for line in @content.split '\n'
leadingWhitespace = /^\s*/.exec(line)[0]
if leadingWhitespace.length > largestIndent.length
largestIndent = leadingWhitespace
@content = @content.replace ///^(#{leadingWhitespace})///gm, ''
if not indent or leadingWhitespace.length < indent.length
indent = leadingWhitespace
@content = @content.replace /// \n #{indent} ///g, '\n' if indent
hasLeadingMarks = /\n\s*[#|\*]/.test @content
@content = @content.replace /^([ \t]*)#(?=\s)/gm, ' *' if hasLeadingMarks
@content = "/*#{@content}#{if hasLeadingMarks then ' ' else ''}*/"
fragment = @makeCode @content
@@ -1509,15 +1532,21 @@ exports.HereComment = class HereComment extends Base
fragment.isComment = fragment.isHereComment = yes
fragment
astType: -> 'CommentBlock'
astProperties: ->
return
value: @content
#### LineComment
# Comment running from `#` to the end of a line (becoming `//`).
exports.LineComment = class LineComment extends Base
constructor: ({ @content, @newLine, @unshift }) ->
constructor: ({ @content, @newLine, @unshift, @locationData, @precededByBlankLine }) ->
super()
compileNode: (o) ->
fragment = @makeCode(if /^\s*$/.test @content then '' else "//#{@content}")
fragment = @makeCode(if /^\s*$/.test @content then '' else "#{if @precededByBlankLine then "\n#{o.indent}" else ''}//#{@content}")
fragment.newLine = @newLine
fragment.unshift = @unshift
fragment.trail = not @newLine and not @unshift
@@ -1525,6 +1554,12 @@ exports.LineComment = class LineComment extends Base
fragment.isComment = fragment.isLineComment = yes
fragment
astType: -> 'CommentLine'
astProperties: ->
return
value: @content
#### JSX
exports.JSXIdentifier = class JSXIdentifier extends IdentifierLiteral
@@ -4756,13 +4791,18 @@ exports.StringWithInterpolations = class StringWithInterpolations extends Base
comment.newLine = yes
attachCommentsToNode salvagedComments, node
if (unwrapped = node.expression?.unwrapAll()) instanceof PassthroughLiteral and unwrapped.generated and not @jsx
commentPlaceholder = new StringLiteral('').withLocationDataFrom node
commentPlaceholder.comments = unwrapped.comments
(commentPlaceholder.comments ?= []).push node.comments... if node.comments
elements.push new Value commentPlaceholder
if o.compiling
commentPlaceholder = new StringLiteral('').withLocationDataFrom node
commentPlaceholder.comments = unwrapped.comments
(commentPlaceholder.comments ?= []).push node.comments... if node.comments
elements.push new Value commentPlaceholder
else
elements.push null
else if node.expression or includeInterpolationWrappers
(node.expression?.comments ?= []).push node.comments... if node.comments
elements.push if includeInterpolationWrappers then node else node.expression
else if not o.compiling
elements.push null
return no
else if node.comments
# This node is getting discarded, but salvage its comments.
@@ -4832,7 +4872,7 @@ exports.StringWithInterpolations = class StringWithInterpolations extends Base
tail: element is last
).withLocationDataFrom(element).ast o
else
expressions.push element.unwrap().ast o
expressions.push element?.unwrap().ast(o) ? null
{expressions, quasis, @quote}

193
test/abstract_syntax_tree.coffee
View File

@@ -44,6 +44,10 @@ testStatement = (code, expected) ->
ast = getAstStatement code
testAgainstExpected ast, expected
testComments = (code, expected) ->
ast = getAstRoot code
testAgainstExpected ast.comments, expected
test 'Confirm functionality of `deepStrictIncludeExpectedProperties`', ->
actual =
name: 'Name'
@@ -132,6 +136,17 @@ test "AST as expected for Block node", ->
directives: []
comments: []
deepStrictIncludeExpectedProperties CoffeeScript.compile('# comment', ast: yes),
type: 'File'
program:
type: 'Program'
body: []
directives: []
comments: [
type: 'CommentLine'
value: ' comment'
]
test "AST as expected for NumberLiteral node", ->
testExpression '42',
type: 'NumericLiteral'
@@ -196,20 +211,23 @@ test "AST as expected for StringLiteral node", ->
]
quote: "'''"
# test "AST as expected for PassthroughLiteral node", ->
# code = 'const CONSTANT = "unreassignable!"'
# testExpression "`#{code}`",
# type: 'PassthroughLiteral'
# value: code
# originalValue: code
# here: no
test "AST as expected for PassthroughLiteral node", ->
code = 'const CONSTANT = "unreassignable!"'
testExpression "`#{code}`",
type: 'PassthroughLiteral'
value: code
here: no
# code = '\nconst CONSTANT = "unreassignable!"\n'
# testExpression "```#{code}```",
# type: 'PassthroughLiteral'
# value: code
# originalValue: code
# here: yes
code = '\nconst CONSTANT = "unreassignable!"\n'
testExpression "```#{code}```",
type: 'PassthroughLiteral'
value: code
here: yes
testExpression "``",
type: 'PassthroughLiteral'
value: ''
here: no
test "AST as expected for IdentifierLiteral node", ->
testExpression 'id',
@@ -637,8 +655,6 @@ test "AST as expected for AwaitReturn node", ->
# # TODO: Figgure out the purpose of `isDefaultValue`. It's not set in `Switch` either.
# # Comments aren’t nodes, so they shouldn’t appear in the AST.
test "AST as expected for Call node", ->
testExpression 'fn()',
type: 'CallExpression'
@@ -2805,6 +2821,47 @@ test "AST as expected for StringWithInterpolations node", ->
]
quote: '"""'
# empty interpolation
testExpression '"#{}"',
type: 'TemplateLiteral'
expressions: [
null
]
quasis: [
type: 'TemplateElement'
value:
raw: ''
tail: no
,
type: 'TemplateElement'
value:
raw: ''
tail: yes
]
quote: '"'
testExpression '''
"#{
# comment
}"
''',
type: 'TemplateLiteral'
expressions: [
null
]
quasis: [
type: 'TemplateElement'
value:
raw: ''
tail: no
,
type: 'TemplateElement'
value:
raw: ''
tail: yes
]
quote: '"'
test "AST as expected for For node", ->
testStatement 'for x, i in arr when x? then return',
type: 'For'
@@ -3578,3 +3635,109 @@ test "AST as expected for directives", ->
expression: ID 'b'
]
directives: []
test "AST as expected for comments", ->
testComments '''
a # simple line comment
''', [
type: 'CommentLine'
value: ' simple line comment'
]
testComments '''
a ### simple here comment ###
''', [
type: 'CommentBlock'
value: ' simple here comment '
]
testComments '''
# just a line comment
''', [
type: 'CommentLine'
value: ' just a line comment'
]
testComments '''
### just a here comment ###
''', [
type: 'CommentBlock'
value: ' just a here comment '
]
testComments '''
"#{
# empty interpolation line comment
}"
''', [
type: 'CommentLine'
value: ' empty interpolation line comment'
]
testComments '''
"#{
### empty interpolation block comment ###
}"
''', [
type: 'CommentBlock'
value: ' empty interpolation block comment '
]
testComments '''
# multiple line comments
# on consecutive lines
''', [
type: 'CommentLine'
value: ' multiple line comments'
,
type: 'CommentLine'
value: ' on consecutive lines'
]
testComments '''
# multiple line comments
# with blank line
''', [
type: 'CommentLine'
value: ' multiple line comments'
,
type: 'CommentLine'
value: ' with blank line'
]
testComments '''
#no whitespace line comment
''', [
type: 'CommentLine'
value: 'no whitespace line comment'
]
testComments '''
###no whitespace here comment###
''', [
type: 'CommentBlock'
value: 'no whitespace here comment'
]
testComments '''
###
# multiline
# here comment
###
''', [
type: 'CommentBlock'
value: '\n# multiline\n# here comment\n'
]
testComments '''
if b
###
# multiline
# indented here comment
###
c
''', [
type: 'CommentBlock'
value: '\n # multiline\n # indented here comment\n '
]

266
test/abstract_syntax_tree_location_data.coffee
View File

@@ -43,6 +43,9 @@ testSingleNodeLocationData = (node, expected, path = '') ->
eq node.loc.end.column, expected.loc.end.column, \
"Expected #{path}.loc.end.column: #{reset}#{node.loc.end.column}#{red} to equal #{reset}#{expected.loc.end.column}#{red}"
testAstCommentsLocationData = (code, expected) ->
testAstNodeLocationData getAstRoot(code).comments, expected
if require?
{mergeAstLocationData, mergeLocationData} = require './../lib/coffeescript/nodes'
@@ -7478,3 +7481,266 @@ test "AST location data as expected for directives", ->
end:
line: 3
column: 7
test "AST location data as expected for PassthroughLiteral node", ->
testAstLocationData "`abc`",
type: 'PassthroughLiteral'
start: 0
end: 5
range: [0, 5]
loc:
start:
line: 1
column: 0
end:
line: 1
column: 5
code = '\nconst CONSTANT = "unreassignable!"\n'
testAstLocationData """
```
abc
```
""",
type: 'PassthroughLiteral'
start: 0
end: 13
range: [0, 13]
loc:
start:
line: 1
column: 0
end:
line: 3
column: 3
testAstLocationData "``",
type: 'PassthroughLiteral'
start: 0
end: 2
range: [0, 2]
loc:
start:
line: 1
column: 0
end:
line: 1
column: 2
test "AST as expected for comments", ->
testAstCommentsLocationData '''
a # simple line comment
''', [
start: 2
end: 23
range: [2, 23]
loc:
start:
line: 1
column: 2
end:
line: 1
column: 23
]
testAstCommentsLocationData '''
a ### simple here comment ###
''', [
start: 2
end: 29
range: [2, 29]
loc:
start:
line: 1
column: 2
end:
line: 1
column: 29
]
testAstCommentsLocationData '''
# just a line comment
''', [
start: 0
end: 21
range: [0, 21]
loc:
start:
line: 1
column: 0
end:
line: 1
column: 21
]
testAstCommentsLocationData '''
### just a here comment ###
''', [
start: 0
end: 27
range: [0, 27]
loc:
start:
line: 1
column: 0
end:
line: 1
column: 27
]
testAstCommentsLocationData '''
"#{
# empty interpolation line comment
}"
''', [
start: 6
end: 40
range: [6, 40]
loc:
start:
line: 2
column: 2
end:
line: 2
column: 36
]
testAstCommentsLocationData '''
"#{
### empty interpolation block comment ###
}"
''', [
start: 6
end: 47
range: [6, 47]
loc:
start:
line: 2
column: 2
end:
line: 2
column: 43
]
testAstCommentsLocationData '''
# multiple line comments
# on consecutive lines
''', [
start: 0
end: 24
range: [0, 24]
loc:
start:
line: 1
column: 0
end:
line: 1
column: 24
,
start: 25
end: 47
range: [25, 47]
loc:
start:
line: 2
column: 0
end:
line: 2
column: 22
]
testAstCommentsLocationData '''
# multiple line comments
# with blank line
''', [
start: 0
end: 24
range: [0, 24]
loc:
start:
line: 1
column: 0
end:
line: 1
column: 24
,
start: 26
end: 43
range: [26, 43]
loc:
start:
line: 3
column: 0
end:
line: 3
column: 17
]
testAstCommentsLocationData '''
#no whitespace line comment
''', [
start: 0
end: 27
range: [0, 27]
loc:
start:
line: 1
column: 0
end:
line: 1
column: 27
]
testAstCommentsLocationData '''
###no whitespace here comment###
''', [
start: 0
end: 32
range: [0, 32]
loc:
start:
line: 1
column: 0
end:
line: 1
column: 32
]
testAstCommentsLocationData '''
###
# multiline
# here comment
###
''', [
start: 0
end: 34
range: [0, 34]
loc:
start:
line: 1
column: 0
end:
line: 4
column: 3
]
testAstCommentsLocationData '''
if b
###
# multiline
# indented here comment
###
c
''', [
start: 7
end: 56
range: [7, 56]
loc:
start:
line: 2
column: 2
end:
line: 5
column: 5
]