Determine the endOfLineInvisibles for each TokenizedLine

spec/tokenized-buffer-spec.coffee

@@ -604,6 +604,26 @@ describe "TokenizedBuffer", ->
       # Also needs to work for copies
       expect(tokenizedBuffer.lineForScreenRow(0).copy().text).toBe "SST Sa line with tabsTand T spacesSTS"
 
+    it "assigns endOfLineInvisibles to tokenized lines", ->
+      buffer = new TextBuffer(text: "a line that ends in a carriage-return-line-feed \r\na line that ends in just a line-feed\na line with no ending")
+      tokenizedBuffer = new TokenizedBuffer({buffer})
+
+      atom.config.set('editor.showInvisibles', true)
+      atom.config.set('editor.invisibles', cr: 'R', eol: 'N')
+      fullyTokenize(tokenizedBuffer)
+
+      expect(tokenizedBuffer.lineForScreenRow(0).endOfLineInvisibles).toEqual ['R', 'N']
+      expect(tokenizedBuffer.lineForScreenRow(1).endOfLineInvisibles).toEqual ['N']
+
+      # Lines ending in soft wraps get no invisibles
+      [left, right] = tokenizedBuffer.lineForScreenRow(0).softWrapAt(20)
+      expect(left.endOfLineInvisibles).toBe null
+      expect(right.endOfLineInvisibles).toEqual ['R', 'N']
+
+      atom.config.set('editor.invisibles', cr: 'R', eol: false)
+      expect(tokenizedBuffer.lineForScreenRow(0).endOfLineInvisibles).toEqual ['R']
+      expect(tokenizedBuffer.lineForScreenRow(1).endOfLineInvisibles).toEqual []
+
   describe "leading and trailing whitespace", ->
     beforeEach ->
       buffer = atom.project.bufferForPathSync('sample.js')

src/tokenized-buffer.coffee

@@ -225,8 +225,9 @@ class TokenizedBuffer extends Model
     tokens = [new Token(value: line, scopes: [@grammar.scopeName])]
     tabLength = @getTabLength()
     indentLevel = @indentLevelForRow(row)
+    lineEnding = @buffer.lineEndingForRow(row)
     invisibles = @invisibles if @showInvisibles
-    new TokenizedLine({tokens, tabLength, indentLevel, invisibles})
+    new TokenizedLine({tokens, tabLength, indentLevel, invisibles, lineEnding})
 
   buildTokenizedTokenizedLineForRow: (row, ruleStack) ->
     line = @buffer.lineForRow(row)

src/tokenized-line.coffee

@@ -8,6 +8,8 @@ idCounter = 1
 
 module.exports =
 class TokenizedLine
+  endOfLineInvisibles: null
+
   constructor: ({tokens, @lineEnding, @ruleStack, @startBufferColumn, @fold, @tabLength, @indentLevel, @invisibles}) ->
     @startBufferColumn ?= 0
     @tokens = @breakOutAtomicTokens(tokens)
@@ -16,7 +18,9 @@ class TokenizedLine
 
     @id = idCounter++
     @markLeadingAndTrailingWhitespaceTokens()
-    @substituteInvisibleCharacters() if @invisibles
+    if @invisibles
+      @substituteInvisibleCharacters()
+      @buildEndOfLineInvisibles() if @lineEnding?
 
   buildText: ->
     text = ""
@@ -95,12 +99,14 @@ class TokenizedLine
       tokens: leftTokens
       startBufferColumn: @startBufferColumn
       ruleStack: @ruleStack
+      invisibles: @invisibles
       lineEnding: null
     )
     rightFragment = new TokenizedLine(
       tokens: rightTokens
       startBufferColumn: @bufferColumnForScreenColumn(column)
       ruleStack: @ruleStack
+      invisibles: @invisibles
       lineEnding: @lineEnding
     )
     [leftFragment, rightFragment]
@@ -174,6 +180,17 @@ class TokenizedLine
 
     @text = @buildText() if changedText
 
+  buildEndOfLineInvisibles: ->
+    @endOfLineInvisibles = []
+    {cr, eol} = @invisibles
+
+    switch @lineEnding
+      when '\r\n'
+        @endOfLineInvisibles.push(cr) if cr
+        @endOfLineInvisibles.push(eol) if eol
+      when '\n'
+        @endOfLineInvisibles.push(eol) if eol
+
   isComment: ->
     for token in @tokens
       continue if token.scopes.length is 1