Handle only buffer coordinates in TokenIterator

spec/tokenized-buffer-spec.coffee

@@ -849,3 +849,27 @@ describe "TokenizedBuffer", ->
 
           iterator.seek(Point(0, 8))
           expect(iterator.getPosition().column).toBe(7)
+
+      it "correctly terminates scopes at the beginning of the line (regression)", ->
+        grammar = atom.grammars.createGrammar('test', {
+          'scopeName': 'text.broken'
+          'name': 'Broken grammar'
+          'patterns': [
+            {'begin': 'start', 'end': '(?=end)', 'name': 'blue.broken'},
+            {'match': '.', 'name': 'yellow.broken'}
+          ]
+        })
+
+        buffer = new TextBuffer(text: 'start x\nend x\nx')
+        tokenizedBuffer = new TokenizedBuffer({
+          buffer, config: atom.config, grammarRegistry: atom.grammars, packageManager: atom.packages, assert: atom.assert
+        })
+        tokenizedBuffer.setGrammar(grammar)
+        fullyTokenize(tokenizedBuffer)
+
+        iterator = tokenizedBuffer.buildIterator()
+        iterator.seek(Point(1, 0))
+
+        expect(iterator.getPosition()).toEqual([1, 0])
+        expect(iterator.getCloseTags()).toEqual ['blue.broken']
+        expect(iterator.getOpenTags()).toEqual ['yellow.broken']

src/lines-tile-component.coffee

@@ -1,7 +1,6 @@
 _ = require 'underscore-plus'
 
 HighlightsComponent = require './highlights-component'
-TokenIterator = require './token-iterator'
 AcceptFilter = {acceptNode: -> NodeFilter.FILTER_ACCEPT}
 TokenTextEscapeRegex = /[&"'<>]/g
 MaxTokenLength = 20000

src/lines-yardstick.coffee

@@ -1,4 +1,3 @@
-TokenIterator = require './token-iterator'
 {Point} = require 'text-buffer'
 {isPairedCharacter} = require './text-utils'
 

src/token-iterator.coffee

@@ -1,72 +1,57 @@
 module.exports =
 class TokenIterator
-  constructor: ({@grammarRegistry}, line, enableScopes) ->
-    @reset(line, enableScopes) if line?
+  constructor: ({@grammarRegistry}, line) ->
+    @reset(line) if line?
 
-  reset: (@line, @enableScopes=true) ->
+  reset: (@line) ->
     @index = null
-    @bufferStart = @line.startBufferColumn
-    @bufferEnd = @bufferStart
-    @screenStart = 0
-    @screenEnd = 0
-    @resetScopes() if @enableScopes
+    @startColumn = 0
+    @endColumn = 0
+    @scopes = @line.openScopes.map (id) => @grammarRegistry.scopeForId(id)
+    @scopeStarts = @scopes.slice()
+    @scopeEnds = []
     this
 
   next: ->
     {tags} = @line
 
     if @index?
+      @startColumn = @endColumn
+      @scopeEnds.length = 0
+      @scopeStarts.length = 0
       @index++
-      @bufferStart = @bufferEnd
-      @screenStart = @screenEnd
-      @clearScopeStartsAndEnds() if @enableScopes
     else
       @index = 0
 
     while @index < tags.length
       tag = tags[@index]
       if tag < 0
-        @handleScopeForTag(tag) if @enableScopes
+        scope = @grammarRegistry.scopeForId(tag)
+        if tag % 2 is 0
+          if @scopeStarts[@scopeStarts.length - 1] is scope
+            @scopeStarts.pop()
+          else
+            @scopeEnds.push(scope)
+          @scopes.pop()
+        else
+          @scopeStarts.push(scope)
+          @scopes.push(scope)
         @index++
       else
-        @screenEnd = @screenStart + tag
-        @bufferEnd = @bufferStart + tag
-
-        @text = @line.text.substring(@screenStart, @screenEnd)
+        @endColumn += tag
+        @text = @line.text.substring(@startColumn, @endColumn)
         return true
 
     false
 
-  resetScopes: ->
-    @scopes = @line.openScopes.map (id) => @grammarRegistry.scopeForId(id)
-    @scopeStarts = @scopes.slice()
-    @scopeEnds = []
-
-  clearScopeStartsAndEnds: ->
-    @scopeEnds.length = 0
-    @scopeStarts.length = 0
-
-  handleScopeForTag: (tag) ->
-    scope = @grammarRegistry.scopeForId(tag)
-    if tag % 2 is 0
-      if @scopeStarts[@scopeStarts.length - 1] is scope
-        @scopeStarts.pop()
-      else
-        @scopeEnds.push(scope)
-      @scopes.pop()
-    else
-      @scopeStarts.push(scope)
-      @scopes.push(scope)
-
-  getBufferStart: -> @bufferStart
-  getBufferEnd: -> @bufferEnd
-
-  getScreenStart: -> @screenStart
-  getScreenEnd: -> @screenEnd
-
-  getScopeStarts: -> @scopeStarts
-  getScopeEnds: -> @scopeEnds
-
   getScopes: -> @scopes
 
+  getScopeStarts: -> @scopeStarts
+
+  getScopeEnds: -> @scopeEnds
+
   getText: -> @text
+
+  getBufferStart: -> @startColumn
+
+  getBufferEnd: -> @endColumn

src/token.coffee

@@ -14,7 +14,7 @@ class Token
 
   isEqual: (other) ->
     # TODO: scopes is deprecated. This is here for the sake of lang package tests
-    @value is other.value and _.isEqual(@scopes, other.scopes) and !!@isAtomic is !!other.isAtomic
+    @value is other.value and _.isEqual(@scopes, other.scopes)
 
   isBracket: ->
     /^meta\.brace\b/.test(_.last(@scopes))