From 0d55a0bd76e1e97b18c96aafeb86646c744bec04 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Wed, 13 Jan 2016 17:54:18 -0700 Subject: [PATCH] Make TokenizedBuffer conform to text decoration layer interface --- spec/tokenized-buffer-spec.coffee | 54 ++++++++++- src/tokenized-buffer-iterator.coffee | 78 +++++++++++++++ src/tokenized-buffer.coffee | 8 ++ src/tokenized-line.coffee | 137 --------------------------- 4 files changed, 139 insertions(+), 138 deletions(-) create mode 100644 src/tokenized-buffer-iterator.coffee diff --git a/spec/tokenized-buffer-spec.coffee b/spec/tokenized-buffer-spec.coffee index 9c53b3c07..4fc0d1618 100644 --- a/spec/tokenized-buffer-spec.coffee +++ b/spec/tokenized-buffer-spec.coffee @@ -1,5 +1,5 @@ TokenizedBuffer = require '../src/tokenized-buffer' -TextBuffer = require 'text-buffer' +{Point} = TextBuffer = require 'text-buffer' _ = require 'underscore-plus' describe "TokenizedBuffer", -> @@ -1061,3 +1061,55 @@ describe "TokenizedBuffer", -> runs -> expect(coffeeCalled).toBe true + + describe "text decoration layer API", -> + describe "iterator", -> + it "iterates over the syntactic scope boundaries", -> + buffer = new TextBuffer(text: "var foo = 1 /*\nhello*/var bar = 2\n") + tokenizedBuffer = new TokenizedBuffer({ + buffer, config: atom.config, grammarRegistry: atom.grammars, packageManager: atom.packages, assert: atom.assert + }) + tokenizedBuffer.setGrammar(atom.grammars.selectGrammar(".js")) + fullyTokenize(tokenizedBuffer) + + iterator = tokenizedBuffer.buildIterator() + iterator.seek(Point(0, 0)) + + boundaries = [] + loop + boundaries.push({ + position: iterator.getPosition(), + closeTags: iterator.getCloseTags(), + openTags: iterator.getOpenTags() + }) + break unless iterator.moveToSuccessor() + + expect(boundaries).toEqual([ + {position: Point(0, 0), closeTags: [], openTags: ["source.js", "storage.type.var.js"]} + {position: Point(0, 3), closeTags: ["storage.type.var.js"], openTags: []} + {position: Point(0, 8), closeTags: [], openTags: ["keyword.operator.assignment.js"]} + {position: Point(0, 9), closeTags: ["keyword.operator.assignment.js"], openTags: []} + {position: Point(0, 10), closeTags: [], openTags: ["constant.numeric.js"]} + {position: Point(0, 11), closeTags: ["constant.numeric.js"], openTags: []} + {position: Point(0, 12), closeTags: [], openTags: ["comment.block.js", "punctuation.definition.comment.js"]} + {position: Point(0, 14), closeTags: ["punctuation.definition.comment.js"], openTags: []} + {position: Point(1, 5), closeTags: [], openTags: ["punctuation.definition.comment.js"]} + {position: Point(1, 7), closeTags: ["punctuation.definition.comment.js", "comment.block.js"], openTags: ["storage.type.var.js"]} + {position: Point(1, 10), closeTags: ["storage.type.var.js"], openTags: []} + {position: Point(1, 15), closeTags: [], openTags: ["keyword.operator.assignment.js"]} + {position: Point(1, 16), closeTags: ["keyword.operator.assignment.js"], openTags: []} + {position: Point(1, 17), closeTags: [], openTags: ["constant.numeric.js"]} + {position: Point(1, 18), closeTags: ["constant.numeric.js"], openTags: []} + ]) + + expect(iterator.seek(Point(0, 1))).toEqual(["source.js", "storage.type.var.js"]) + expect(iterator.getPosition()).toEqual(Point(0, 3)) + expect(iterator.seek(Point(0, 8))).toEqual(["source.js"]) + expect(iterator.getPosition()).toEqual(Point(0, 8)) + expect(iterator.seek(Point(1, 0))).toEqual(["source.js", "comment.block.js"]) + expect(iterator.getPosition()).toEqual(Point(1, 5)) + expect(iterator.seek(Point(1, 18))).toEqual(["source.js", "constant.numeric.js"]) + expect(iterator.getPosition()).toEqual(Point(1, 18)) + + expect(iterator.seek(Point(2, 0))).toEqual(["source.js"]) + iterator.moveToSuccessor() # ensure we don't infinitely loop (regression test) diff --git a/src/tokenized-buffer-iterator.coffee b/src/tokenized-buffer-iterator.coffee new file mode 100644 index 000000000..acafb8f4b --- /dev/null +++ b/src/tokenized-buffer-iterator.coffee @@ -0,0 +1,78 @@ +{Point} = require 'text-buffer' + +module.exports = +class TokenizedBufferIterator + constructor: (@tokenizedBuffer, @grammarRegistry) -> + @openTags = null + @closeTags = null + + seek: (position) -> + @openTags = [] + @closeTags = [] + @tagIndex = null + + currentLine = @tokenizedBuffer.tokenizedLineForRow(position.row) + containingTags = currentLine.openScopes.map (id) => @grammarRegistry.scopeForId(id) + @currentTags = currentLine.tags + currentColumn = 0 + for tag, index in @currentTags + if tag >= 0 + if currentColumn >= position.column and @isAtTagBoundary() + @tagIndex = index + break + else + currentColumn += tag + containingTags.pop() while @closeTags.shift() + containingTags.push(tag) while tag = @openTags.shift() + else + scopeName = @grammarRegistry.scopeForId(tag) + if tag % 2 is 0 + @closeTags.push(scopeName) + else + @openTags.push(scopeName) + + @tagIndex ?= @currentTags.length + @position = Point(position.row, currentColumn) + containingTags + + moveToSuccessor: -> + if @tagIndex is @currentTags.length + @position = Point(@position.row + 1, 0) + @currentTags = @tokenizedBuffer.tokenizedLineForRow(@position.row)?.tags + return false unless @currentTags? + @tagIndex = 0 + else + @position = Point(@position.row, @position.column + @currentTags[@tagIndex]) + @tagIndex++ + + @openTags = [] + @closeTags = [] + + loop + tag = @currentTags[@tagIndex] + if tag >= 0 or @tagIndex is @currentTags.length + if @isAtTagBoundary() + break + else + return @moveToSuccessor() + else + scopeName = @grammarRegistry.scopeForId(tag) + if tag % 2 is 0 + @closeTags.push(scopeName) + else + @openTags.push(scopeName) + @tagIndex++ + + true + + getPosition: -> + @position + + getCloseTags: -> + @closeTags.slice() + + getOpenTags: -> + @openTags.slice() + + isAtTagBoundary: -> + @closeTags.length > 0 or @openTags.length > 0 diff --git a/src/tokenized-buffer.coffee b/src/tokenized-buffer.coffee index 0fdf4eea8..0267e02dd 100644 --- a/src/tokenized-buffer.coffee +++ b/src/tokenized-buffer.coffee @@ -7,6 +7,7 @@ TokenizedLine = require './tokenized-line' TokenIterator = require './token-iterator' Token = require './token' ScopeDescriptor = require './scope-descriptor' +TokenizedBufferIterator = require './tokenized-buffer-iterator' module.exports = class TokenizedBuffer extends Model @@ -58,6 +59,12 @@ class TokenizedBuffer extends Model destroyed: -> @disposables.dispose() + buildIterator: -> + new TokenizedBufferIterator(this, @grammarRegistry) + + getInvalidatedRanges: -> + [@invalidatedRange] + serialize: -> state = { deserializer: 'TokenizedBuffer' @@ -274,6 +281,7 @@ class TokenizedBuffer extends Model [start, end] = @updateFoldableStatus(start, end + delta) end -= delta + @invalidatedRange = Range(start, end) event = {start, end, delta, bufferChange: e} @emitter.emit 'did-change', event diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index c97a621ac..e063387eb 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -47,143 +47,6 @@ class TokenizedLine @startBufferColumn ?= 0 @bufferDelta = @text.length - @transformContent() - @buildEndOfLineInvisibles() if @invisibles? and @lineEnding? - - transformContent: -> - text = '' - bufferColumn = 0 - screenColumn = 0 - tokenIndex = 0 - tokenOffset = 0 - firstNonWhitespaceColumn = null - lastNonWhitespaceColumn = null - - substringStart = 0 - substringEnd = 0 - - while bufferColumn < @text.length - # advance to next token if we've iterated over its length - if tokenOffset is @tags[tokenIndex] - tokenIndex++ - tokenOffset = 0 - - # advance to next token tag - tokenIndex++ while @tags[tokenIndex] < 0 - - charCode = @text.charCodeAt(bufferColumn) - - # split out unicode surrogate pairs - if isPairedCharacter(@text, bufferColumn) - prefix = tokenOffset - suffix = @tags[tokenIndex] - tokenOffset - 2 - - i = tokenIndex - @tags.splice(i, 1) - @tags.splice(i++, 0, prefix) if prefix > 0 - @tags.splice(i++, 0, 2) - @tags.splice(i, 0, suffix) if suffix > 0 - - firstNonWhitespaceColumn ?= screenColumn - lastNonWhitespaceColumn = screenColumn + 1 - - substringEnd += 2 - screenColumn += 2 - bufferColumn += 2 - - tokenIndex++ if prefix > 0 - @specialTokens[tokenIndex] = PairedCharacter - tokenIndex++ - tokenOffset = 0 - - # split out leading soft tabs - else if charCode is SpaceCharCode - if firstNonWhitespaceColumn? - substringEnd += 1 - else - if (screenColumn + 1) % @tabLength is 0 - suffix = @tags[tokenIndex] - @tabLength - if suffix >= 0 - @specialTokens[tokenIndex] = SoftTab - @tags.splice(tokenIndex, 1, @tabLength) - @tags.splice(tokenIndex + 1, 0, suffix) if suffix > 0 - - if @invisibles?.space - if substringEnd > substringStart - text += @text.substring(substringStart, substringEnd) - substringStart = substringEnd - text += @invisibles.space - substringStart += 1 - - substringEnd += 1 - - screenColumn++ - bufferColumn++ - tokenOffset++ - - # expand hard tabs to the next tab stop - else if charCode is TabCharCode - if substringEnd > substringStart - text += @text.substring(substringStart, substringEnd) - substringStart = substringEnd - - tabLength = @tabLength - (screenColumn % @tabLength) - if @invisibles?.tab - text += @invisibles.tab - text += getTabString(tabLength - 1) if tabLength > 1 - else - text += getTabString(tabLength) - - substringStart += 1 - substringEnd += 1 - - prefix = tokenOffset - suffix = @tags[tokenIndex] - tokenOffset - 1 - - i = tokenIndex - @tags.splice(i, 1) - @tags.splice(i++, 0, prefix) if prefix > 0 - @tags.splice(i++, 0, tabLength) - @tags.splice(i, 0, suffix) if suffix > 0 - - screenColumn += tabLength - bufferColumn++ - - tokenIndex++ if prefix > 0 - @specialTokens[tokenIndex] = HardTab - tokenIndex++ - tokenOffset = 0 - - # continue past any other character - else - firstNonWhitespaceColumn ?= screenColumn - lastNonWhitespaceColumn = screenColumn - - substringEnd += 1 - screenColumn++ - bufferColumn++ - tokenOffset++ - - if substringEnd > substringStart - unless substringStart is 0 and substringEnd is @text.length - text += @text.substring(substringStart, substringEnd) - @text = text - else - @text = text - - @firstNonWhitespaceIndex = firstNonWhitespaceColumn - if lastNonWhitespaceColumn? - if lastNonWhitespaceColumn + 1 < @text.length - @firstTrailingWhitespaceIndex = lastNonWhitespaceColumn + 1 - if @invisibles?.space - @text = - @text.substring(0, @firstTrailingWhitespaceIndex) + - @text.substring(@firstTrailingWhitespaceIndex) - .replace(RepeatedSpaceRegex, @invisibles.space) - else - @lineIsWhitespaceOnly = true - @firstTrailingWhitespaceIndex = 0 - getTokenIterator: -> @tokenIterator.reset(this, arguments...) Object.defineProperty @prototype, 'tokens', get: ->