Create tokens in TokenizedBuffer

This commit is contained in:
Kevin Sawicki
2013-12-31 10:57:51 -08:00
parent c1fc09e510
commit e1aec57ffe
2 changed files with 1 additions and 2 deletions

View File

@@ -203,6 +203,7 @@ class TokenizedBuffer extends Model
lineEnding = @buffer.lineEndingForRow(row)
tabLength = @getTabLength()
{ tokens, ruleStack } = @grammar.tokenizeLine(line, ruleStack, row is 0)
tokens = (new Token(token) for token in tokens)
new TokenizedLine({tokens, ruleStack, tabLength, lineEnding})
# FIXME: benogle says: These are actually buffer rows as all buffer rows are

View File

@@ -1,12 +1,10 @@
_ = require 'underscore-plus'
Token = require './token'
### Internal ###
module.exports =
class TokenizedLine
constructor: ({tokens, @lineEnding, @ruleStack, @startBufferColumn, @fold, tabLength}) ->
tokens = (new Token(token) for token in tokens)
@tokens = @breakOutAtomicTokens(tokens, tabLength)
@startBufferColumn ?= 0
@text = _.pluck(@tokens, 'value').join('')