Mark tokens with leading/trailing whitespace when building TokenizedLine

spec/tokenized-buffer-spec.coffee

@@ -463,3 +463,40 @@ describe "TokenizedBuffer", ->
       expect(tokenizedBuffer.tokenForPosition([0,0]).value).toBe ' '
       atom.config.set('editor.tabLength', 0)
       expect(tokenizedBuffer.tokenForPosition([0,0]).value).toBe '  '
+
+  describe "leading and trailing whitespace", ->
+    beforeEach ->
+      buffer = atom.project.bufferForPathSync('sample.js')
+      tokenizedBuffer = new TokenizedBuffer({buffer})
+      fullyTokenize(tokenizedBuffer)
+
+    it "sets ::hasLeadingWhitespace to true on tokens that have leading whitespace", ->
+      expect(tokenizedBuffer.lineForScreenRow(0).tokens[0].hasLeadingWhitespace).toBe false
+      expect(tokenizedBuffer.lineForScreenRow(1).tokens[0].hasLeadingWhitespace).toBe true
+      expect(tokenizedBuffer.lineForScreenRow(1).tokens[1].hasLeadingWhitespace).toBe false
+      expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].hasLeadingWhitespace).toBe true
+      expect(tokenizedBuffer.lineForScreenRow(2).tokens[1].hasLeadingWhitespace).toBe true
+      expect(tokenizedBuffer.lineForScreenRow(2).tokens[2].hasLeadingWhitespace).toBe false
+
+      # The 4th token *has* leading whitespace, but isn't entirely whitespace
+      buffer.insert([5, 0], ' ')
+      expect(tokenizedBuffer.lineForScreenRow(5).tokens[3].hasLeadingWhitespace).toBe true
+      expect(tokenizedBuffer.lineForScreenRow(5).tokens[4].hasLeadingWhitespace).toBe false
+
+      # Lines that are *only* whitespace are not considered to have leading whitespace
+      buffer.insert([10, 0], '  ')
+      expect(tokenizedBuffer.lineForScreenRow(10).tokens[0].hasLeadingWhitespace).toBe false
+
+    it "sets ::hasTrailingWhitespace to true on tokens that have trailing whitespace", ->
+      buffer.insert([0, Infinity], '  ')
+      expect(tokenizedBuffer.lineForScreenRow(0).tokens[11].hasTrailingWhitespace).toBe false
+      expect(tokenizedBuffer.lineForScreenRow(0).tokens[12].hasTrailingWhitespace).toBe true
+
+      # The last token *has* trailing whitespace, but isn't entirely whitespace
+      buffer.setTextInRange([[2, 39], [2, 40]], '  ')
+      expect(tokenizedBuffer.lineForScreenRow(2).tokens[14].hasTrailingWhitespace).toBe false
+      expect(tokenizedBuffer.lineForScreenRow(2).tokens[15].hasTrailingWhitespace).toBe true
+
+      # Lines that are *only* whitespace are considered to have trailing whitespace
+      buffer.insert([10, 0], '  ')
+      expect(tokenizedBuffer.lineForScreenRow(10).tokens[0].hasTrailingWhitespace).toBe true

src/token.coffee

@@ -20,6 +20,8 @@ class Token
   scopes: null
   isAtomic: null
   isHardTab: null
+  hasLeadingWhitespace: false
+  hasTrailingWhitespace: false
 
   constructor: ({@value, @scopes, @isAtomic, @bufferDelta, @isHardTab}) ->
     @screenDelta = @value.length

src/tokenized-line.coffee

@@ -10,6 +10,7 @@ class TokenizedLine
     @text = _.pluck(@tokens, 'value').join('')
     @bufferDelta = _.sum(_.pluck(@tokens, 'bufferDelta'))
     @id = idCounter++
+    @markLeadingAndTrailingWhitespaceTokens()
 
   copy: ->
     new TokenizedLine({@tokens, @lineEnding, @ruleStack, @startBufferColumn, @fold})
@@ -117,6 +118,16 @@ class TokenizedLine
       breakOutLeadingSoftTabs = token.isOnlyWhitespace() if breakOutLeadingSoftTabs
     outputTokens
 
+  markLeadingAndTrailingWhitespaceTokens: ->
+    firstNonWhitespacePosition = @text.search(/\S/)
+    firstTrailingWhitespacePosition = @text.search(/\s*$/)
+    lineIsWhitespaceOnly = firstTrailingWhitespacePosition is 0
+    position = 0
+    for token, i in @tokens
+      token.hasLeadingWhitespace =  position < firstNonWhitespacePosition
+      token.hasTrailingWhitespace = position + token.value.length > firstTrailingWhitespacePosition
+      position += token.value.length
+
   isComment: ->
     for token in @tokens
       continue if token.scopes.length is 1