diff --git a/src/tokenized-buffer.coffee b/src/tokenized-buffer.coffee index ac3cf01b9..e747d3062 100644 --- a/src/tokenized-buffer.coffee +++ b/src/tokenized-buffer.coffee @@ -203,6 +203,7 @@ class TokenizedBuffer extends Model lineEnding = @buffer.lineEndingForRow(row) tabLength = @getTabLength() { tokens, ruleStack } = @grammar.tokenizeLine(line, ruleStack, row is 0) + tokens = (new Token(token) for token in tokens) new TokenizedLine({tokens, ruleStack, tabLength, lineEnding}) # FIXME: benogle says: These are actually buffer rows as all buffer rows are diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index 26d24fb0b..826fdfd5c 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -1,12 +1,10 @@ _ = require 'underscore-plus' -Token = require './token' ### Internal ### module.exports = class TokenizedLine constructor: ({tokens, @lineEnding, @ruleStack, @startBufferColumn, @fold, tabLength}) -> - tokens = (new Token(token) for token in tokens) @tokens = @breakOutAtomicTokens(tokens, tabLength) @startBufferColumn ?= 0 @text = _.pluck(@tokens, 'value').join('')