Pass TokenIterator reference instead of using global singleton

Adds an extra reference to each tokenized line but is also more sane.

/cc @maxbrunsfeld

src/display-buffer.coffee

@@ -4,7 +4,6 @@ Serializable = require 'serializable'
 {Point, Range} = require 'text-buffer'
 Grim = require 'grim'
 TokenizedBuffer = require './tokenized-buffer'
-TokenIterator = require './token-iterator'
 RowMap = require './row-map'
 Fold = require './fold'
 Model = require './model'
@@ -652,7 +651,7 @@ class DisplayBuffer extends Model
     left = 0
     column = 0
 
-    iterator = TokenIterator.instance.reset(@tokenizedLineForScreenRow(targetRow))
+    iterator = @tokenizedLineForScreenRow(targetRow).getTokenIterator()
     while iterator.next()
       charWidths = @getScopedCharWidths(iterator.getScopes())
       valueIndex = 0
@@ -685,7 +684,7 @@ class DisplayBuffer extends Model
     left = 0
     column = 0
 
-    iterator = TokenIterator.instance.reset(@tokenizedLineForScreenRow(row))
+    iterator = @tokenizedLineForScreenRow(row).getTokenIterator()
     while iterator.next()
       charWidths = @getScopedCharWidths(iterator.getScopes())
       value = iterator.getText()

src/language-mode.coffee

@@ -2,7 +2,6 @@
 _ = require 'underscore-plus'
 {OnigRegExp} = require 'oniguruma'
 ScopeDescriptor = require './scope-descriptor'
-TokenIterator = require './token-iterator'
 
 module.exports =
 class LanguageMode
@@ -243,7 +242,7 @@ class LanguageMode
     @suggestedIndentForTokenizedLineAtBufferRow(bufferRow, tokenizedLine, options)
 
   suggestedIndentForTokenizedLineAtBufferRow: (bufferRow, tokenizedLine, options) ->
-    iterator = TokenIterator.instance.reset(tokenizedLine)
+    iterator = tokenizedLine.getTokenIterator()
     iterator.next()
     scopeDescriptor = new ScopeDescriptor(scopes: iterator.getScopes())
 

src/lines-component.coffee

@@ -22,6 +22,7 @@ class LinesComponent
   placeholderTextDiv: null
 
   constructor: ({@presenter, @hostElement, @useShadowDOM, visible}) ->
+    @tokenIterator = new TokenIterator
     @measuredLines = new Set
     @lineNodesByLineId = {}
     @screenRowsByLineId = {}
@@ -175,19 +176,19 @@ class LinesComponent
     lineIsWhitespaceOnly = firstTrailingWhitespaceIndex is 0
 
     innerHTML = ""
-    iterator = TokenIterator.instance.reset(lineState)
+    @tokenIterator.reset(lineState)
 
-    while iterator.next()
-      for scope in iterator.getScopeEnds()
+    while @tokenIterator.next()
+      for scope in @tokenIterator.getScopeEnds()
         innerHTML += "</span>"
 
-      for scope in iterator.getScopeStarts()
+      for scope in @tokenIterator.getScopeStarts()
         innerHTML += "<span class=\"#{scope.replace(/\.+/g, ' ')}\">"
 
-      tokenStart = iterator.getScreenStart()
-      tokenEnd = iterator.getScreenEnd()
-      tokenText = iterator.getText()
-      isHardTab = iterator.isHardTab()
+      tokenStart = @tokenIterator.getScreenStart()
+      tokenEnd = @tokenIterator.getScreenEnd()
+      tokenText = @tokenIterator.getText()
+      isHardTab = @tokenIterator.isHardTab()
 
       if hasLeadingWhitespace = tokenStart < firstNonWhitespaceIndex
         tokenFirstNonWhitespaceIndex = firstNonWhitespaceIndex - tokenStart
@@ -209,10 +210,10 @@ class LinesComponent
 
       innerHTML += @buildTokenHTML(tokenText, isHardTab, tokenFirstNonWhitespaceIndex, tokenFirstTrailingWhitespaceIndex, hasIndentGuide, hasInvisibleCharacters)
 
-    for scope in iterator.getScopeEnds()
+    for scope in @tokenIterator.getScopeEnds()
       innerHTML += "</span>"
 
-    for scope in iterator.getScopes()
+    for scope in @tokenIterator.getScopes()
       innerHTML += "</span>"
 
     innerHTML += @buildEndOfLineHTML(id)
@@ -353,15 +354,15 @@ class LinesComponent
     iterator = null
     charIndex = 0
 
-    tokenIterator = TokenIterator.instance.reset(tokenizedLine)
-    while tokenIterator.next()
-      scopes = tokenIterator.getScopes()
-      text = tokenIterator.getText()
+    @tokenIterator.reset(tokenizedLine)
+    while @tokenIterator.next()
+      scopes = @tokenIterator.getScopes()
+      text = @tokenIterator.getText()
       charWidths = @presenter.getScopedCharacterWidths(scopes)
 
       textIndex = 0
       while textIndex < text.length
-        if tokenIterator.isPairedCharacter()
+        if @tokenIterator.isPairedCharacter()
           char = text
           charLength = 2
           textIndex += 2

src/text-editor-presenter.coffee

@@ -2,7 +2,6 @@
 {Point, Range} = require 'text-buffer'
 _ = require 'underscore-plus'
 Decoration = require './decoration'
-TokenIterator = require './token-iterator'
 
 module.exports =
 class TextEditorPresenter
@@ -1013,7 +1012,7 @@ class TextEditorPresenter
     left = 0
     column = 0
 
-    iterator = TokenIterator.instance.reset(@model.tokenizedLineForScreenRow(targetRow))
+    iterator = @model.tokenizedLineForScreenRow(targetRow).getTokenIterator()
     while iterator.next()
       characterWidths = @getScopedCharacterWidths(iterator.getScopes())
 

src/token-iterator.coffee

@@ -81,5 +81,3 @@ class TokenIterator
 
   isAtomic: ->
     @isSoftTab() or @isHardTab() or @isSoftWrapIndentation() or @isPairedCharacter()
-
-TokenIterator.instance = new TokenIterator

src/tokenized-buffer.coffee

@@ -27,6 +27,7 @@ class TokenizedBuffer extends Model
   constructor: ({@buffer, @tabLength, @ignoreInvisibles}) ->
     @emitter = new Emitter
     @disposables = new CompositeDisposable
+    @tokenIterator = new TokenIterator
 
     @disposables.add atom.grammars.onDidAddGrammar(@grammarAddedOrUpdated)
     @disposables.add atom.grammars.onDidUpdateGrammar(@grammarAddedOrUpdated)
@@ -321,7 +322,7 @@ class TokenizedBuffer extends Model
     tabLength = @getTabLength()
     indentLevel = @indentLevelForRow(row)
     lineEnding = @buffer.lineEndingForRow(row)
-    new TokenizedLine({openScopes, text, tags, tabLength, indentLevel, invisibles: @getInvisiblesToShow(), lineEnding})
+    new TokenizedLine({openScopes, text, tags, tabLength, indentLevel, invisibles: @getInvisiblesToShow(), lineEnding, @tokenIterator})
 
   buildTokenizedLineForRow: (row, ruleStack, openScopes) ->
     @buildTokenizedLineForRowWithText(row, @buffer.lineForRow(row), ruleStack, openScopes)
@@ -331,7 +332,7 @@ class TokenizedBuffer extends Model
     tabLength = @getTabLength()
     indentLevel = @indentLevelForRow(row)
     {tags, ruleStack} = @grammar.tokenizeLine(text, ruleStack, row is 0, false)
-    new TokenizedLine({openScopes, text, tags, ruleStack, tabLength, lineEnding, indentLevel, invisibles: @getInvisiblesToShow()})
+    new TokenizedLine({openScopes, text, tags, ruleStack, tabLength, lineEnding, indentLevel, invisibles: @getInvisiblesToShow(), @tokenIterator})
 
   getInvisiblesToShow: ->
     if @configSettings.showInvisibles and not @ignoreInvisibles
@@ -402,7 +403,7 @@ class TokenizedBuffer extends Model
   scopeDescriptorForPosition: (position) ->
     {row, column} = Point.fromObject(position)
 
-    iterator = TokenIterator.instance.reset(@tokenizedLines[row])
+    iterator = @tokenizedLines[row].getTokenIterator()
     while iterator.next()
       if iterator.getScreenEnd() > column
         scopes = iterator.getScopes()

src/tokenized-line.coffee

@@ -1,7 +1,6 @@
 _ = require 'underscore-plus'
 {isPairedCharacter} = require './text-utils'
 Token = require './token'
-TokenIterator = require './token-iterator'
 {SoftTab, HardTab, PairedCharacter, SoftWrapIndent} = require './special-token-symbols'
 
 NonWhitespaceRegex = /\S/
@@ -24,7 +23,7 @@ class TokenizedLine
     return unless properties?
 
     @specialTokens = {}
-    {@openScopes, @text, @tags, @lineEnding, @ruleStack} = properties
+    {@openScopes, @text, @tags, @lineEnding, @ruleStack, @tokenIterator} = properties
     {@startBufferColumn, @fold, @tabLength, @indentLevel, @invisibles} = properties
 
     @startBufferColumn ?= 0
@@ -143,7 +142,7 @@ class TokenizedLine
       @lineIsWhitespaceOnly = true
       @firstTrailingWhitespaceIndex = 0
 
-  getTokenIterator: -> TokenIterator.instance.reset(this)
+  getTokenIterator: -> @tokenIterator.reset(this)
 
   Object.defineProperty @prototype, 'tokens', get: ->
     iterator = @getTokenIterator()
@@ -179,6 +178,7 @@ class TokenizedLine
 
   copy: ->
     copy = new TokenizedLine
+    copy.tokenIterator = @tokenIterator
     copy.indentLevel = @indentLevel
     copy.openScopes = @openScopes
     copy.text = @text
@@ -359,6 +359,7 @@ class TokenizedLine
     splitBufferColumn = @bufferColumnForScreenColumn(column)
 
     leftFragment = new TokenizedLine
+    leftFragment.tokenIterator = @tokenIterator
     leftFragment.openScopes = @openScopes
     leftFragment.text = leftText
     leftFragment.tags = leftTags
@@ -374,6 +375,7 @@ class TokenizedLine
     leftFragment.firstTrailingWhitespaceIndex = Math.min(column, @firstTrailingWhitespaceIndex)
 
     rightFragment = new TokenizedLine
+    rightFragment.tokenIterator = @tokenIterator
     rightFragment.openScopes = rightOpenScopes
     rightFragment.text = rightText
     rightFragment.tags = rightTags