Mark leading/trailing whitespace in new TokenizedLine representation

src/token.coffee

@@ -20,7 +20,11 @@ class Token
   firstTrailingWhitespaceIndex: null
   hasInvisibleCharacters: false
 
-  constructor: ({@value, @scopes, @isAtomic, @bufferDelta, @isHardTab, @hasPairedCharacter, @isSoftWrapIndentation}) ->
+  constructor: (properties) ->
+    {@value, @scopes, @isAtomic, @bufferDelta, @isHardTab, @hasPairedCharacter, @isSoftWrapIndentation} = properties
+    @firstNonWhitespaceIndex = properties.firstNonWhitespaceIndex ? null
+    @firstTrailingWhitespaceIndex = properties.firstTrailingWhitespaceIndex ? null
+    
     @screenDelta = @value.length
     @bufferDelta ?= @screenDelta
     @hasPairedCharacter ?= textUtils.hasPairedCharacter(@value)

src/tokenized-line.coffee

@@ -122,6 +122,8 @@ class TokenizedLine
     @text = text
 
   Object.defineProperty @prototype, 'tokens', get: ->
+    offset = 0
+
     atom.grammars.decodeContent @text, @tags, @parentScopes.slice(), (tokenProperties, index) =>
       switch @specialTokens[index]
         when SoftTab
@@ -133,6 +135,16 @@ class TokenizedLine
           tokenProperties.isAtomic = true
           tokenProperties.hasPairedCharacter = true
 
+      if offset < @firstNonWhitespaceIndex
+        tokenProperties.firstNonWhitespaceIndex =
+          Math.min(offset + tokenProperties.value.length, @firstNonWhitespaceIndex - offset)
+
+      if @lineEnding? and (offset + tokenProperties.value.length > @firstTrailingWhitespaceIndex)
+        tokenProperties.firstTrailingWhitespaceIndex =
+          Math.max(0, @firstTrailingWhitespaceIndex - offset)
+
+      offset += tokenProperties.value.length
+
       new Token(tokenProperties)
 
   buildText: ->
@@ -332,17 +344,8 @@ class TokenizedLine
     @firstNonWhitespaceIndex = @text.search(NonWhitespaceRegex)
     if @firstNonWhitespaceIndex > 0 and isPairedCharacter(@text, @firstNonWhitespaceIndex - 1)
       @firstNonWhitespaceIndex--
-    firstTrailingWhitespaceIndex = @text.search(TrailingWhitespaceRegex)
-    @lineIsWhitespaceOnly = firstTrailingWhitespaceIndex is 0
-    index = 0
-    for token in @tokens
-      if index < @firstNonWhitespaceIndex
-        token.firstNonWhitespaceIndex = Math.min(index + token.value.length, @firstNonWhitespaceIndex - index)
-      # Only the *last* segment of a soft-wrapped line can have trailing whitespace
-      if @lineEnding? and (index + token.value.length > firstTrailingWhitespaceIndex)
-        token.firstTrailingWhitespaceIndex = Math.max(0, firstTrailingWhitespaceIndex - index)
-      index += token.value.length
-    return
+    @firstTrailingWhitespaceIndex = @text.search(TrailingWhitespaceRegex)
+    @lineIsWhitespaceOnly = @firstTrailingWhitespaceIndex is 0
 
   substituteInvisibleCharacters: ->
     invisibles = @invisibles