safer lexing at the start of the files

This commit is contained in:
Jeremy Ashkenas
2010-02-12 21:04:33 -05:00
parent 207ec81821
commit b21780b738
2 changed files with 7 additions and 7 deletions

View File

@@ -194,14 +194,15 @@
};
// Record tokens for indentation differing from the previous line.
lex.prototype.indent_token = function indent_token() {
var diff, indent, next_character, no_newlines, size;
var diff, indent, next_character, no_newlines, prev, size;
if (!((indent = this.match(MULTI_DENT, 1)))) {
return false;
}
this.line += indent.match(MULTILINER).length;
this.i += indent.length;
next_character = this.chunk.match(MULTI_DENT)[4];
no_newlines = next_character === '.' || (this.value().match(NO_NEWLINE) && this.tokens[this.tokens.length - 2][0] !== '.' && !this.value().match(CODE));
prev = this.tokens[this.tokens.length - 2];
no_newlines = next_character === '.' || (this.value().match(NO_NEWLINE) && prev && (prev[0] !== '.') && !this.value().match(CODE));
if (no_newlines) {
return this.suppress_newlines(indent);
}
@@ -228,9 +229,7 @@
this.token('OUTDENT', last_indent);
move_out -= last_indent;
}
if (!(this.tag() === 'TERMINATOR')) {
this.token('TERMINATOR', "\n");
}
this.token('TERMINATOR', "\n");
return true;
};
// Matches and consumes non-meaningful whitespace.