Add a TokenIterator and use it for tokens shim

src/token-iterator.coffee

@@ -0,0 +1,85 @@
+{SoftTab, HardTab, PairedCharacter, SoftWrapIndent} = require './special-token-symbols'
+
+module.exports =
+class TokenIterator
+  @instance: new this
+
+  constructor: (line) ->
+    @reset(line) if line?
+
+  reset: (@line) ->
+    @index = null
+    @bufferStart = 0
+    @bufferEnd = 0
+    @screenStart = 0
+    @screenEnd = 0
+    @scopes = @line.openScopes.map (id) -> atom.grammars.scopeForId(id)
+    @scopeStarts = @scopes.slice()
+    @scopeEnds = []
+    this
+
+  next: ->
+    {tags} = @line
+
+    if @index?
+      @index++
+      @scopeEnds.length = 0
+      @scopeStarts.length = 0
+      @bufferStart = @bufferEnd
+      @screenStart = @screenEnd
+    else
+      @index = 0
+
+    while @index < tags.length
+      tag = tags[@index]
+      if tag < 0
+        if tag % 2 is 0
+          @scopeEnds.push(atom.grammars.scopeForId(tag + 1))
+          @scopes.pop()
+        else
+          scope = atom.grammars.scopeForId(tag)
+          @scopeStarts.push(scope)
+          @scopes.push(scope)
+        @index++
+      else
+        if @isHardTab()
+          @screenEnd = @screenStart + tag
+          @bufferEnd = @bufferStart + 1
+        else if @isSoftWrapIndentation()
+          @screenEnd = @screenStart + tag
+          @bufferEnd = @bufferStart + 0
+        else
+          @screenEnd = @screenStart + tag
+          @bufferEnd = @bufferStart + tag
+        return true
+
+    false
+
+  getBufferStart: -> @bufferStart
+  getBufferEnd: -> @bufferEnd
+
+  getScreenStart: -> @screenStart
+  getScreenEnd: -> @screenEnd
+
+  getScopeStarts: -> @scopeStarts
+  getScopeEnds: -> @scopeEnds
+
+  getScopes: -> @scopes
+
+  getText: ->
+    @line.text.substring(@screenStart, @screenEnd)
+
+  isSoftTab: ->
+    @line.specialTokens[@index] is SoftTab
+
+  isHardTab: ->
+    @line.specialTokens[@index] is HardTab
+
+  isSoftWrapIndentation: ->
+    @line.specialTokens[@index] is SoftWrapIndent
+
+  isPairedCharacter: ->
+    @line.specialTokens[@index] is PairedCharacter
+
+  isAtomic: ->
+    @isSoftTab() or @isHardTab() or @isSoftWrapIndentation() or @isPairedCharacter()

src/tokenized-line.coffee

@@ -1,6 +1,7 @@
 _ = require 'underscore-plus'
 {isPairedCharacter} = require './text-utils'
 Token = require './token'
+TokenIterator = require './token-iterator'
 {SoftTab, HardTab, PairedCharacter, SoftWrapIndent} = require './special-token-symbols'
 
 NonWhitespaceRegex = /\S/
@@ -28,10 +29,10 @@ class TokenizedLine
     @startBufferColumn ?= 0
     @bufferDelta = @text.length
 
-    @subdivideTokens()
+    @transformContent()
     @buildEndOfLineInvisibles() if @invisibles? and @lineEnding?
 
-  subdivideTokens: ->
+  transformContent: ->
     text = ''
     bufferColumn = 0
     screenColumn = 0
@@ -142,37 +143,36 @@ class TokenizedLine
       @firstTrailingWhitespaceIndex = 0
 
   Object.defineProperty @prototype, 'tokens', get: ->
-    offset = 0
+    iterator = TokenIterator.instance.reset(this)
+    tokens = []
 
-    atom.grammars.decodeTokens @text, @tags, @openScopes.slice(), (tokenProperties, index) =>
-      switch @specialTokens[index]
-        when SoftTab
-          tokenProperties.isAtomic = true
-        when HardTab
-          tokenProperties.isAtomic = true
-          tokenProperties.isHardTab = true
-          tokenProperties.bufferDelta = 1
-          tokenProperties.hasInvisibleCharacters = true if @invisibles?.tab
-        when PairedCharacter
-          tokenProperties.isAtomic = true
-          tokenProperties.hasPairedCharacter = true
-        when SoftWrapIndent
-          tokenProperties.isAtomic = true
-          tokenProperties.isSoftWrapIndentation = true
+    while iterator.next()
+      properties = {
+        value: iterator.getText()
+        scopes: iterator.getScopes().slice()
+        isAtomic: iterator.isAtomic()
+        isHardTab: iterator.isHardTab()
+        hasPairedCharacter: iterator.isPairedCharacter()
+        isSoftWrapIndentation: iterator.isSoftWrapIndentation()
+      }
 
-      if offset < @firstNonWhitespaceIndex
-        tokenProperties.firstNonWhitespaceIndex =
-          Math.min(tokenProperties.value.length, @firstNonWhitespaceIndex - offset)
-        tokenProperties.hasInvisibleCharacters = true if @invisibles?.space
+      if iterator.isHardTab()
+        properties.bufferDelta = 1
+        properties.hasInvisibleCharacters = true if @invisibles?.tab
 
-      if @lineEnding? and (offset + tokenProperties.value.length > @firstTrailingWhitespaceIndex)
-        tokenProperties.firstTrailingWhitespaceIndex =
-          Math.max(0, @firstTrailingWhitespaceIndex - offset)
-        tokenProperties.hasInvisibleCharacters = true if @invisibles?.space
+      if iterator.getScreenStart() < @firstNonWhitespaceIndex
+        properties.firstNonWhitespaceIndex =
+          Math.min(@firstNonWhitespaceIndex, iterator.getScreenEnd()) - iterator.getScreenStart()
+        properties.hasInvisibleCharacters = true if @invisibles?.space
 
-      offset += tokenProperties.value.length
+      if @lineEnding? and iterator.getScreenEnd() > @firstTrailingWhitespaceIndex
+        properties.firstTrailingWhitespaceIndex =
+          Math.max(0, @firstTrailingWhitespaceIndex - iterator.getScreenStart())
+        properties.hasInvisibleCharacters = true if @invisibles?.space
 
-      new Token(tokenProperties)
+      tokens.push(new Token(properties))
+
+    tokens
 
   copy: ->
     copy = new TokenizedLine