diff --git a/spec/tokenized-buffer-iterator-spec.js b/spec/tokenized-buffer-iterator-spec.js index 8d0e458f4..93aaf21e0 100644 --- a/spec/tokenized-buffer-iterator-spec.js +++ b/spec/tokenized-buffer-iterator-spec.js @@ -4,100 +4,152 @@ import TokenizedBufferIterator from '../src/tokenized-buffer-iterator' import {Point} from 'text-buffer' describe('TokenizedBufferIterator', () => { - it('reports two boundaries at the same position when tags close, open, then close again without a non-negative integer separating them (regression)', () => { - const tokenizedBuffer = { - tokenizedLineForRow () { - return { - tags: [-1, -2, -1, -2], - text: '', - openScopes: [] - } - } - } - - const grammarRegistry = { - scopeForId () { - return 'foo' - } - } - - const iterator = new TokenizedBufferIterator(tokenizedBuffer, grammarRegistry) - - iterator.seek(Point(0, 0)) - expect(iterator.getPosition()).toEqual(Point(0, 0)) - expect(iterator.getCloseTags()).toEqual([]) - expect(iterator.getOpenTags()).toEqual(['foo']) - - iterator.moveToSuccessor() - expect(iterator.getPosition()).toEqual(Point(0, 0)) - expect(iterator.getCloseTags()).toEqual(['foo']) - expect(iterator.getOpenTags()).toEqual(['foo']) - - iterator.moveToSuccessor() - expect(iterator.getCloseTags()).toEqual(['foo']) - expect(iterator.getOpenTags()).toEqual([]) - }) - - it("reports a boundary at line end if the next line's open scopes don't match the containing tags for the current line", () => { - const tokenizedBuffer = { - tokenizedLineForRow (row) { - if (row === 0) { + describe('seek(position)', function () { + it('seeks to the leftmost tag boundary at the given position, returning the containing tags', function () { + const tokenizedBuffer = { + tokenizedLineForRow (row) { return { - tags: [-1, 3, -2, -3], - text: 'bar', + tags: [-1, -2, -3, -4, -5, 3, -3, -4, -6], + text: 'foo', openScopes: [] } - } else if (row === 1) { + } + } + + const grammarRegistry = { + scopeForId (id) { return { - tags: [3], - text: 'baz', - openScopes: [-1] - } - } else if (row === 2) { + '-1': 'foo', '-2': 'foo', + '-3': 'bar', '-4': 'bar', + '-5': 'baz', '-6': 'baz' + }[id] + } + } + + const iterator = new TokenizedBufferIterator(tokenizedBuffer, grammarRegistry) + + expect(iterator.seek(Point(0, 0))).toEqual([]) + expect(iterator.getCloseTags()).toEqual([]) + expect(iterator.getOpenTags()).toEqual(['foo']) + + iterator.moveToSuccessor() + expect(iterator.getCloseTags()).toEqual(['foo']) + expect(iterator.getOpenTags()).toEqual(['bar']) + + expect(iterator.seek(Point(0, 1))).toEqual(['baz']) + expect(iterator.getCloseTags()).toEqual([]) + expect(iterator.getOpenTags()).toEqual([]) + + iterator.moveToSuccessor() + expect(iterator.getCloseTags()).toEqual([]) + expect(iterator.getOpenTags()).toEqual(['bar']) + + expect(iterator.seek(Point(0, 3))).toEqual(['baz']) + expect(iterator.getCloseTags()).toEqual([]) + expect(iterator.getOpenTags()).toEqual(['bar']) + + iterator.moveToSuccessor() + expect(iterator.getCloseTags()).toEqual(['bar', 'baz']) + expect(iterator.getOpenTags()).toEqual([]) + }) + }) + + describe('moveToSuccessor()', function () { + it('reports two boundaries at the same position when tags close, open, then close again without a non-negative integer separating them (regression)', () => { + const tokenizedBuffer = { + tokenizedLineForRow () { return { - tags: [-2], + tags: [-1, -2, -1, -2], text: '', - openScopes: [-1] + openScopes: [] } } } - } - const grammarRegistry = { - scopeForId (id) { - if (id === -2 || id === -1) { + const grammarRegistry = { + scopeForId () { return 'foo' - } else if (id === -3) { - return 'qux' } } - } - const iterator = new TokenizedBufferIterator(tokenizedBuffer, grammarRegistry) + const iterator = new TokenizedBufferIterator(tokenizedBuffer, grammarRegistry) - iterator.seek(Point(0, 0)) - expect(iterator.getPosition()).toEqual(Point(0, 0)) - expect(iterator.getCloseTags()).toEqual([]) - expect(iterator.getOpenTags()).toEqual(['foo']) + iterator.seek(Point(0, 0)) + expect(iterator.getPosition()).toEqual(Point(0, 0)) + expect(iterator.getCloseTags()).toEqual([]) + expect(iterator.getOpenTags()).toEqual(['foo']) - iterator.moveToSuccessor() - expect(iterator.getPosition()).toEqual(Point(0, 3)) - expect(iterator.getCloseTags()).toEqual(['foo']) - expect(iterator.getOpenTags()).toEqual(['qux']) + iterator.moveToSuccessor() + expect(iterator.getPosition()).toEqual(Point(0, 0)) + expect(iterator.getCloseTags()).toEqual(['foo']) + expect(iterator.getOpenTags()).toEqual(['foo']) - iterator.moveToSuccessor() - expect(iterator.getPosition()).toEqual(Point(0, 3)) - expect(iterator.getCloseTags()).toEqual(['qux']) - expect(iterator.getOpenTags()).toEqual([]) + iterator.moveToSuccessor() + expect(iterator.getCloseTags()).toEqual(['foo']) + expect(iterator.getOpenTags()).toEqual([]) + }) - iterator.moveToSuccessor() - expect(iterator.getPosition()).toEqual(Point(1, 0)) - expect(iterator.getCloseTags()).toEqual([]) - expect(iterator.getOpenTags()).toEqual(['foo']) + it("reports a boundary at line end if the next line's open scopes don't match the containing tags for the current line", () => { + const tokenizedBuffer = { + tokenizedLineForRow (row) { + if (row === 0) { + return { + tags: [-1, 3, -2, -3], + text: 'bar', + openScopes: [] + } + } else if (row === 1) { + return { + tags: [3], + text: 'baz', + openScopes: [-1] + } + } else if (row === 2) { + return { + tags: [-2], + text: '', + openScopes: [-1] + } + } + } + } - iterator.moveToSuccessor() - expect(iterator.getPosition()).toEqual(Point(2, 0)) - expect(iterator.getCloseTags()).toEqual(['foo']) - expect(iterator.getOpenTags()).toEqual([]) + const grammarRegistry = { + scopeForId (id) { + if (id === -2 || id === -1) { + return 'foo' + } else if (id === -3) { + return 'qux' + } + } + } + + const iterator = new TokenizedBufferIterator(tokenizedBuffer, grammarRegistry) + + iterator.seek(Point(0, 0)) + expect(iterator.getPosition()).toEqual(Point(0, 0)) + expect(iterator.getCloseTags()).toEqual([]) + expect(iterator.getOpenTags()).toEqual(['foo']) + + iterator.moveToSuccessor() + expect(iterator.getPosition()).toEqual(Point(0, 3)) + expect(iterator.getCloseTags()).toEqual(['foo']) + expect(iterator.getOpenTags()).toEqual(['qux']) + + iterator.moveToSuccessor() + expect(iterator.getPosition()).toEqual(Point(0, 3)) + expect(iterator.getCloseTags()).toEqual(['qux']) + expect(iterator.getOpenTags()).toEqual([]) + + iterator.moveToSuccessor() + expect(iterator.getPosition()).toEqual(Point(1, 0)) + expect(iterator.getCloseTags()).toEqual([]) + expect(iterator.getOpenTags()).toEqual(['foo']) + + iterator.moveToSuccessor() + expect(iterator.getPosition()).toEqual(Point(2, 0)) + expect(iterator.getCloseTags()).toEqual(['foo']) + expect(iterator.getOpenTags()).toEqual([]) + }) }) }) diff --git a/spec/tokenized-buffer-spec.coffee b/spec/tokenized-buffer-spec.coffee index d11c93213..6c437d953 100644 --- a/spec/tokenized-buffer-spec.coffee +++ b/spec/tokenized-buffer-spec.coffee @@ -816,7 +816,7 @@ describe "TokenizedBuffer", -> expect(iterator.seek(Point(0, 8))).toEqual(["source.js"]) expect(iterator.getPosition()).toEqual(Point(0, 8)) expect(iterator.seek(Point(1, 0))).toEqual(["source.js", "comment.block.js"]) - expect(iterator.getPosition()).toEqual(Point(1, 5)) + expect(iterator.getPosition()).toEqual(Point(1, 0)) expect(iterator.seek(Point(1, 18))).toEqual(["source.js", "constant.numeric.decimal.js"]) expect(iterator.getPosition()).toEqual(Point(1, 18)) diff --git a/src/tokenized-buffer-iterator.coffee b/src/tokenized-buffer-iterator.coffee index 780156e42..591943a48 100644 --- a/src/tokenized-buffer-iterator.coffee +++ b/src/tokenized-buffer-iterator.coffee @@ -18,24 +18,31 @@ class TokenizedBufferIterator @currentLineLength = currentLine.text.length @containingTags = @currentLineOpenTags.map (id) => @grammarRegistry.scopeForId(id) currentColumn = 0 + for tag, index in @currentTags if tag >= 0 - if currentColumn >= position.column and @isAtTagBoundary() + if currentColumn is position.column @tagIndex = index break else currentColumn += tag @containingTags.pop() while @closeTags.shift() - @containingTags.push(tag) while tag = @openTags.shift() - else - scopeName = @grammarRegistry.scopeForId(tag) - if tag % 2 is 0 - if @openTags.length > 0 + @containingTags.push(openTag) while openTag = @openTags.shift() + if currentColumn > position.column @tagIndex = index break - else - @closeTags.push(scopeName) - else + else + scopeName = @grammarRegistry.scopeForId(tag) + if tag % 2 is 0 # close tag + if @openTags.length > 0 + if currentColumn is position.column + @tagIndex = index + break + else + @containingTags.pop() while @closeTags.shift() + @containingTags.push(openTag) while openTag = @openTags.shift() + @closeTags.push(scopeName) + else # open tag @openTags.push(scopeName) @tagIndex ?= @currentTags.length