Merge pull request #12289 from atom/ns-as-fix-tokenized-buffer-iterator

Always seek to specified position in TokenizedBufferIterator

spec/tokenized-buffer-iterator-spec.js

@@ -4,100 +4,152 @@ import TokenizedBufferIterator from '../src/tokenized-buffer-iterator'
 import {Point} from 'text-buffer'
 
 describe('TokenizedBufferIterator', () => {
-  it('reports two boundaries at the same position when tags close, open, then close again without a non-negative integer separating them (regression)', () => {
-    const tokenizedBuffer = {
-      tokenizedLineForRow () {
-        return {
-          tags: [-1, -2, -1, -2],
-          text: '',
-          openScopes: []
-        }
-      }
-    }
-
-    const grammarRegistry = {
-      scopeForId () {
-        return 'foo'
-      }
-    }
-
-    const iterator = new TokenizedBufferIterator(tokenizedBuffer, grammarRegistry)
-
-    iterator.seek(Point(0, 0))
-    expect(iterator.getPosition()).toEqual(Point(0, 0))
-    expect(iterator.getCloseTags()).toEqual([])
-    expect(iterator.getOpenTags()).toEqual(['foo'])
-
-    iterator.moveToSuccessor()
-    expect(iterator.getPosition()).toEqual(Point(0, 0))
-    expect(iterator.getCloseTags()).toEqual(['foo'])
-    expect(iterator.getOpenTags()).toEqual(['foo'])
-
-    iterator.moveToSuccessor()
-    expect(iterator.getCloseTags()).toEqual(['foo'])
-    expect(iterator.getOpenTags()).toEqual([])
-  })
-
-  it("reports a boundary at line end if the next line's open scopes don't match the containing tags for the current line", () => {
-    const tokenizedBuffer = {
-      tokenizedLineForRow (row) {
-        if (row === 0) {
+  describe('seek(position)', function () {
+    it('seeks to the leftmost tag boundary at the given position, returning the containing tags', function () {
+      const tokenizedBuffer = {
+        tokenizedLineForRow (row) {
           return {
-            tags: [-1, 3, -2, -3],
-            text: 'bar',
+            tags: [-1, -2, -3, -4, -5, 3, -3, -4, -6],
+            text: 'foo',
             openScopes: []
           }
-        } else if (row === 1) {
+        }
+      }
+
+      const grammarRegistry = {
+        scopeForId (id) {
           return {
-            tags: [3],
-            text: 'baz',
-            openScopes: [-1]
-          }
-        } else if (row === 2) {
+            '-1': 'foo', '-2': 'foo',
+            '-3': 'bar', '-4': 'bar',
+            '-5': 'baz', '-6': 'baz'
+          }[id]
+        }
+      }
+
+      const iterator = new TokenizedBufferIterator(tokenizedBuffer, grammarRegistry)
+
+      expect(iterator.seek(Point(0, 0))).toEqual([])
+      expect(iterator.getCloseTags()).toEqual([])
+      expect(iterator.getOpenTags()).toEqual(['foo'])
+
+      iterator.moveToSuccessor()
+      expect(iterator.getCloseTags()).toEqual(['foo'])
+      expect(iterator.getOpenTags()).toEqual(['bar'])
+
+      expect(iterator.seek(Point(0, 1))).toEqual(['baz'])
+      expect(iterator.getCloseTags()).toEqual([])
+      expect(iterator.getOpenTags()).toEqual([])
+
+      iterator.moveToSuccessor()
+      expect(iterator.getCloseTags()).toEqual([])
+      expect(iterator.getOpenTags()).toEqual(['bar'])
+
+      expect(iterator.seek(Point(0, 3))).toEqual(['baz'])
+      expect(iterator.getCloseTags()).toEqual([])
+      expect(iterator.getOpenTags()).toEqual(['bar'])
+
+      iterator.moveToSuccessor()
+      expect(iterator.getCloseTags()).toEqual(['bar', 'baz'])
+      expect(iterator.getOpenTags()).toEqual([])
+    })
+  })
+
+  describe('moveToSuccessor()', function () {
+    it('reports two boundaries at the same position when tags close, open, then close again without a non-negative integer separating them (regression)', () => {
+      const tokenizedBuffer = {
+        tokenizedLineForRow () {
           return {
-            tags: [-2],
+            tags: [-1, -2, -1, -2],
             text: '',
-            openScopes: [-1]
+            openScopes: []
           }
         }
       }
-    }
 
-    const grammarRegistry = {
-      scopeForId (id) {
-        if (id === -2 || id === -1) {
+      const grammarRegistry = {
+        scopeForId () {
           return 'foo'
-        } else if (id === -3) {
-          return 'qux'
         }
       }
-    }
 
-    const iterator = new TokenizedBufferIterator(tokenizedBuffer, grammarRegistry)
+      const iterator = new TokenizedBufferIterator(tokenizedBuffer, grammarRegistry)
 
-    iterator.seek(Point(0, 0))
-    expect(iterator.getPosition()).toEqual(Point(0, 0))
-    expect(iterator.getCloseTags()).toEqual([])
-    expect(iterator.getOpenTags()).toEqual(['foo'])
+      iterator.seek(Point(0, 0))
+      expect(iterator.getPosition()).toEqual(Point(0, 0))
+      expect(iterator.getCloseTags()).toEqual([])
+      expect(iterator.getOpenTags()).toEqual(['foo'])
 
-    iterator.moveToSuccessor()
-    expect(iterator.getPosition()).toEqual(Point(0, 3))
-    expect(iterator.getCloseTags()).toEqual(['foo'])
-    expect(iterator.getOpenTags()).toEqual(['qux'])
+      iterator.moveToSuccessor()
+      expect(iterator.getPosition()).toEqual(Point(0, 0))
+      expect(iterator.getCloseTags()).toEqual(['foo'])
+      expect(iterator.getOpenTags()).toEqual(['foo'])
 
-    iterator.moveToSuccessor()
-    expect(iterator.getPosition()).toEqual(Point(0, 3))
-    expect(iterator.getCloseTags()).toEqual(['qux'])
-    expect(iterator.getOpenTags()).toEqual([])
+      iterator.moveToSuccessor()
+      expect(iterator.getCloseTags()).toEqual(['foo'])
+      expect(iterator.getOpenTags()).toEqual([])
+    })
 
-    iterator.moveToSuccessor()
-    expect(iterator.getPosition()).toEqual(Point(1, 0))
-    expect(iterator.getCloseTags()).toEqual([])
-    expect(iterator.getOpenTags()).toEqual(['foo'])
+    it("reports a boundary at line end if the next line's open scopes don't match the containing tags for the current line", () => {
+      const tokenizedBuffer = {
+        tokenizedLineForRow (row) {
+          if (row === 0) {
+            return {
+              tags: [-1, 3, -2, -3],
+              text: 'bar',
+              openScopes: []
+            }
+          } else if (row === 1) {
+            return {
+              tags: [3],
+              text: 'baz',
+              openScopes: [-1]
+            }
+          } else if (row === 2) {
+            return {
+              tags: [-2],
+              text: '',
+              openScopes: [-1]
+            }
+          }
+        }
+      }
 
-    iterator.moveToSuccessor()
-    expect(iterator.getPosition()).toEqual(Point(2, 0))
-    expect(iterator.getCloseTags()).toEqual(['foo'])
-    expect(iterator.getOpenTags()).toEqual([])
+      const grammarRegistry = {
+        scopeForId (id) {
+          if (id === -2 || id === -1) {
+            return 'foo'
+          } else if (id === -3) {
+            return 'qux'
+          }
+        }
+      }
+
+      const iterator = new TokenizedBufferIterator(tokenizedBuffer, grammarRegistry)
+
+      iterator.seek(Point(0, 0))
+      expect(iterator.getPosition()).toEqual(Point(0, 0))
+      expect(iterator.getCloseTags()).toEqual([])
+      expect(iterator.getOpenTags()).toEqual(['foo'])
+
+      iterator.moveToSuccessor()
+      expect(iterator.getPosition()).toEqual(Point(0, 3))
+      expect(iterator.getCloseTags()).toEqual(['foo'])
+      expect(iterator.getOpenTags()).toEqual(['qux'])
+
+      iterator.moveToSuccessor()
+      expect(iterator.getPosition()).toEqual(Point(0, 3))
+      expect(iterator.getCloseTags()).toEqual(['qux'])
+      expect(iterator.getOpenTags()).toEqual([])
+
+      iterator.moveToSuccessor()
+      expect(iterator.getPosition()).toEqual(Point(1, 0))
+      expect(iterator.getCloseTags()).toEqual([])
+      expect(iterator.getOpenTags()).toEqual(['foo'])
+
+      iterator.moveToSuccessor()
+      expect(iterator.getPosition()).toEqual(Point(2, 0))
+      expect(iterator.getCloseTags()).toEqual(['foo'])
+      expect(iterator.getOpenTags()).toEqual([])
+    })
   })
 })

spec/tokenized-buffer-spec.coffee

@@ -816,7 +816,7 @@ describe "TokenizedBuffer", ->
         expect(iterator.seek(Point(0, 8))).toEqual(["source.js"])
         expect(iterator.getPosition()).toEqual(Point(0, 8))
         expect(iterator.seek(Point(1, 0))).toEqual(["source.js", "comment.block.js"])
-        expect(iterator.getPosition()).toEqual(Point(1, 5))
+        expect(iterator.getPosition()).toEqual(Point(1, 0))
         expect(iterator.seek(Point(1, 18))).toEqual(["source.js", "constant.numeric.decimal.js"])
         expect(iterator.getPosition()).toEqual(Point(1, 18))
 

src/tokenized-buffer-iterator.coffee

@@ -18,24 +18,31 @@ class TokenizedBufferIterator
     @currentLineLength = currentLine.text.length
     @containingTags = @currentLineOpenTags.map (id) => @grammarRegistry.scopeForId(id)
     currentColumn = 0
+
     for tag, index in @currentTags
       if tag >= 0
-        if currentColumn >= position.column and @isAtTagBoundary()
+        if currentColumn is position.column
           @tagIndex = index
           break
         else
           currentColumn += tag
           @containingTags.pop() while @closeTags.shift()
-          @containingTags.push(tag) while tag = @openTags.shift()
-      else
-        scopeName = @grammarRegistry.scopeForId(tag)
-        if tag % 2 is 0
-          if @openTags.length > 0
+          @containingTags.push(openTag) while openTag = @openTags.shift()
+          if currentColumn > position.column
             @tagIndex = index
             break
-          else
-            @closeTags.push(scopeName)
-        else
+      else
+        scopeName = @grammarRegistry.scopeForId(tag)
+        if tag % 2 is 0 # close tag
+          if @openTags.length > 0
+            if currentColumn is position.column
+              @tagIndex = index
+              break
+            else
+              @containingTags.pop() while @closeTags.shift()
+              @containingTags.push(openTag) while openTag = @openTags.shift()
+          @closeTags.push(scopeName)
+        else # open tag
           @openTags.push(scopeName)
 
     @tagIndex ?= @currentTags.length