Break out atomic tokens in ScreenLine instead of TextMateGrammar

This paves the way for creating untokenized screen lines that still have leading whitespace and tab characters broken out appropriately.

spec/app/text-mate-grammar-spec.coffee

@@ -10,7 +10,7 @@ describe "TextMateGrammar", ->
   beforeEach ->
     grammar = TextMateBundle.grammarForFilePath("hello.coffee")
 
-  describe ".tokenizeLine(line, { ruleStack, tabLength })", ->
+  describe ".tokenizeLine(line, ruleStack)", ->
     describe "when the entire line matches a single pattern with no capture groups", ->
       it "returns a single token with the correct scope", ->
         {tokens} = grammar.tokenizeLine("return")
@@ -142,7 +142,7 @@ describe "TextMateGrammar", ->
       describe "when the pattern spans multiple lines", ->
         it "uses the ruleStack returned by the first line to parse the second line", ->
           {tokens: firstTokens, ruleStack} = grammar.tokenizeLine("'''single-quoted")
-          {tokens: secondTokens, ruleStack} = grammar.tokenizeLine("heredoc'''", {ruleStack})
+          {tokens: secondTokens, ruleStack} = grammar.tokenizeLine("heredoc'''", ruleStack)
 
           expect(firstTokens.length).toBe 2
           expect(secondTokens.length).toBe 2

src/app/screen-line.coffee

@@ -2,7 +2,8 @@ _ = require 'underscore'
 
 module.exports =
 class ScreenLine
-  constructor: ({@tokens, @ruleStack, @bufferRows, @startBufferColumn, @fold}) ->
+  constructor: ({tokens, @ruleStack, @bufferRows, @startBufferColumn, @fold, tabLength}) ->
+    @tokens = @breakOutAtomicTokens(tokens, tabLength)
     @bufferRows ?= 1
     @startBufferColumn ?= 0
     @text = _.pluck(@tokens, 'value').join('')
@@ -90,3 +91,11 @@ class ScreenLine
       delta += token.bufferDelta
       return token if delta >= bufferColumn
     token
+
+  breakOutAtomicTokens: (inputTokens, tabLength) ->
+    outputTokens = []
+    breakOutLeadingWhitespace = true
+    for token in inputTokens
+      outputTokens.push(token.breakOutAtomicTokens(tabLength, breakOutLeadingWhitespace)...)
+      breakOutLeadingWhitespace = token.isOnlyWhitespace() if breakOutLeadingWhitespace
+    outputTokens

src/app/text-mate-grammar.coffee

@@ -29,7 +29,7 @@ class TextMateGrammar
       data = {patterns: [data], tempName: name} if data.begin? or data.match?
       @repository[name] = new Rule(this, data)
 
-  tokenizeLine: (line, {ruleStack, tabLength}={}) ->
+  tokenizeLine: (line, ruleStack=[@initialRule]) ->
     ruleStack ?= [@initialRule]
     ruleStack = new Array(ruleStack...) # clone ruleStack
     tokens = []
@@ -62,15 +62,7 @@ class TextMateGrammar
         ))
         break
 
-    { tokens: @breakOutAtomicTokens(tokens, tabLength), ruleStack }
-
-  breakOutAtomicTokens: (inputTokens, tabLength) ->
-    outputTokens = []
-    breakOutLeadingWhitespace = true
-    for token in inputTokens
-      outputTokens.push(token.breakOutAtomicTokens(tabLength, breakOutLeadingWhitespace)...)
-      breakOutLeadingWhitespace = token.isOnlyWhitespace() if breakOutLeadingWhitespace
-    outputTokens
+    { tokens, ruleStack }
 
   ruleForInclude: (name) ->
     if name[0] == "#"

src/app/tokenized-buffer.coffee

@@ -66,7 +66,8 @@ class TokenizedBuffer
 
   buildScreenLineForRow: (row, ruleStack) ->
     line = @buffer.lineForRow(row)
-    new ScreenLine(@languageMode.tokenizeLine(line, {ruleStack, @tabLength}))
+    { tokens, ruleStack } = @languageMode.tokenizeLine(line, ruleStack)
+    new ScreenLine({tokens, ruleStack, @tabLength})
 
   lineForScreenRow: (row) ->
     @screenLines[row]