Break out atomic tokens in ScreenLine instead of TextMateGrammar

This paves the way for creating untokenized screen lines that still have leading whitespace and tab characters broken out appropriately.

src/app/screen-line.coffee

@@ -2,7 +2,8 @@ _ = require 'underscore'
 
 module.exports =
 class ScreenLine
-  constructor: ({@tokens, @ruleStack, @bufferRows, @startBufferColumn, @fold}) ->
+  constructor: ({tokens, @ruleStack, @bufferRows, @startBufferColumn, @fold, tabLength}) ->
+    @tokens = @breakOutAtomicTokens(tokens, tabLength)
     @bufferRows ?= 1
     @startBufferColumn ?= 0
     @text = _.pluck(@tokens, 'value').join('')
@@ -90,3 +91,11 @@ class ScreenLine
       delta += token.bufferDelta
       return token if delta >= bufferColumn
     token
+
+  breakOutAtomicTokens: (inputTokens, tabLength) ->
+    outputTokens = []
+    breakOutLeadingWhitespace = true
+    for token in inputTokens
+      outputTokens.push(token.breakOutAtomicTokens(tabLength, breakOutLeadingWhitespace)...)
+      breakOutLeadingWhitespace = token.isOnlyWhitespace() if breakOutLeadingWhitespace
+    outputTokens

src/app/text-mate-grammar.coffee

@@ -29,7 +29,7 @@ class TextMateGrammar
       data = {patterns: [data], tempName: name} if data.begin? or data.match?
       @repository[name] = new Rule(this, data)
 
-  tokenizeLine: (line, {ruleStack, tabLength}={}) ->
+  tokenizeLine: (line, ruleStack=[@initialRule]) ->
     ruleStack ?= [@initialRule]
     ruleStack = new Array(ruleStack...) # clone ruleStack
     tokens = []
@@ -62,15 +62,7 @@ class TextMateGrammar
         ))
         break
 
-    { tokens: @breakOutAtomicTokens(tokens, tabLength), ruleStack }
-
-  breakOutAtomicTokens: (inputTokens, tabLength) ->
-    outputTokens = []
-    breakOutLeadingWhitespace = true
-    for token in inputTokens
-      outputTokens.push(token.breakOutAtomicTokens(tabLength, breakOutLeadingWhitespace)...)
-      breakOutLeadingWhitespace = token.isOnlyWhitespace() if breakOutLeadingWhitespace
-    outputTokens
+    { tokens, ruleStack }
 
   ruleForInclude: (name) ->
     if name[0] == "#"

src/app/tokenized-buffer.coffee

@@ -66,7 +66,8 @@ class TokenizedBuffer
 
   buildScreenLineForRow: (row, ruleStack) ->
     line = @buffer.lineForRow(row)
-    new ScreenLine(@languageMode.tokenizeLine(line, {ruleStack, @tabLength}))
+    { tokens, ruleStack } = @languageMode.tokenizeLine(line, ruleStack)
+    new ScreenLine({tokens, ruleStack, @tabLength})
 
   lineForScreenRow: (row) ->
     @screenLines[row]