Set invisible values for spaces and tabs when initial tokenization occurs.

Also break whitespace into its own token just like tabs.

src/app/edit-session.coffee

@@ -44,7 +44,7 @@ class EditSession
     @id = @constructor.idCounter++
     @softTabs ?= true
     @languageMode = new LanguageMode(this, @buffer.getExtension())
-    @displayBuffer = new DisplayBuffer(@buffer, { @languageMode, @tabLength })
+    @displayBuffer = new DisplayBuffer(@buffer, { @languageMode, @tabLength, @showInvisibles })
     @tokenizedBuffer = @displayBuffer.tokenizedBuffer
     @anchors = []
     @anchorRanges = []

src/app/token.coffee

@@ -5,7 +5,6 @@ class Token
   value: null
   scopes: null
   isAtomic: null
-  isTab: null
 
   constructor: ({@value, @scopes, @isAtomic, @bufferDelta, @fold, @isTab}) ->
     @screenDelta = @value.length
@@ -22,13 +21,19 @@ class Token
     value2 = @value.substring(splitIndex)
     [new Token(value: value1, scopes: @scopes), new Token(value: value2, scopes: @scopes)]
 
-  breakOutTabCharacters: (tabLength) ->
-    return [this] unless /\t/.test(@value)
+  breakOutWhitespaceCharacters: (tabLength, showInvisibles) ->
+    return [this] unless /\t| /.test(@value)
 
-    tabText = new Array(tabLength + 1).join(" ")
-    for substring in @value.match(/([^\t]+|\t)/g)
-      if substring == '\t'
-        new Token(value: tabText, scopes: @scopes, bufferDelta: 1, isAtomic: true, isTab: true)
+    for substring in @value.match(/([^\t ]+|(\t| +))/g)
+      scopesForInvisibles = if showInvisibles then @scopes.concat("invisible") else @scopes
+
+      if substring == "\t"
+        value = new Array(tabLength + 1).join(" ")
+        value = "▸" + value[1..] if showInvisibles
+        new Token(value: value, scopes: scopesForInvisibles, bufferDelta: 1, isAtomic: true)
+      else if /^ +$/.test(substring)
+        value = if showInvisibles then substring.replace(/[ ]/g, "•") else substring
+        new Token(value: value, scopes: scopesForInvisibles)
       else
         new Token(value: substring, scopes: @scopes)
 
@@ -45,10 +50,4 @@ class Token
       .replace(/</g, '&lt;')
       .replace(/>/g, '&gt;')
 
-    if showInvisibles
-      if @isTab
-        value = "▸" + value[1..]
-      else
-        value = value.replace(/[ ]+/g, "•")
-
     value

src/app/tokenized-buffer.coffee

@@ -11,11 +11,12 @@ class TokenizedBuffer
 
   languageMode: null
   tabLength: null
+  showInvisibles: null
   buffer: null
   aceAdaptor: null
   screenLines: null
 
-  constructor: (@buffer, { @languageMode, @tabLength }) ->
+  constructor: (@buffer, { @languageMode, @tabLength, @showInvisibles }) ->
     @tabLength ?= 2
     @languageMode.tokenizedBuffer = this
     @id = @constructor.idCounter++
@@ -66,7 +67,7 @@ class TokenizedBuffer
     tokenObjects = []
     for tokenProperties in tokens
       token = new Token(tokenProperties)
-      tokenObjects.push(token.breakOutTabCharacters(@tabLength)...)
+      tokenObjects.push(token.breakOutWhitespaceCharacters(@tabLength, @showInvisibles)...)
     text = _.pluck(tokenObjects, 'value').join('')
     new ScreenLine(tokenObjects, text, [1, 0], [1, 0], { stack })