mirror of
https://github.com/atom/atom.git
synced 2026-01-24 14:28:14 -05:00
Make TokenizedBuffer conform to text decoration layer interface
This commit is contained in:
Nathan Sobo
@@ -1,5 +1,5 @@
|
||||
TokenizedBuffer = require '../src/tokenized-buffer'
|
||||
TextBuffer = require 'text-buffer'
|
||||
{Point} = TextBuffer = require 'text-buffer'
|
||||
_ = require 'underscore-plus'
|
||||
|
||||
describe "TokenizedBuffer", ->
|
||||
@@ -1061,3 +1061,55 @@ describe "TokenizedBuffer", ->
|
||||
|
||||
runs ->
|
||||
expect(coffeeCalled).toBe true
|
||||
|
||||
describe "text decoration layer API", ->
|
||||
describe "iterator", ->
|
||||
it "iterates over the syntactic scope boundaries", ->
|
||||
buffer = new TextBuffer(text: "var foo = 1 /*\nhello*/var bar = 2\n")
|
||||
tokenizedBuffer = new TokenizedBuffer({
|
||||
buffer, config: atom.config, grammarRegistry: atom.grammars, packageManager: atom.packages, assert: atom.assert
|
||||
})
|
||||
tokenizedBuffer.setGrammar(atom.grammars.selectGrammar(".js"))
|
||||
fullyTokenize(tokenizedBuffer)
|
||||
|
||||
iterator = tokenizedBuffer.buildIterator()
|
||||
iterator.seek(Point(0, 0))
|
||||
|
||||
boundaries = []
|
||||
loop
|
||||
boundaries.push({
|
||||
position: iterator.getPosition(),
|
||||
closeTags: iterator.getCloseTags(),
|
||||
openTags: iterator.getOpenTags()
|
||||
})
|
||||
break unless iterator.moveToSuccessor()
|
||||
|
||||
expect(boundaries).toEqual([
|
||||
{position: Point(0, 0), closeTags: [], openTags: ["source.js", "storage.type.var.js"]}
|
||||
{position: Point(0, 3), closeTags: ["storage.type.var.js"], openTags: []}
|
||||
{position: Point(0, 8), closeTags: [], openTags: ["keyword.operator.assignment.js"]}
|
||||
{position: Point(0, 9), closeTags: ["keyword.operator.assignment.js"], openTags: []}
|
||||
{position: Point(0, 10), closeTags: [], openTags: ["constant.numeric.js"]}
|
||||
{position: Point(0, 11), closeTags: ["constant.numeric.js"], openTags: []}
|
||||
{position: Point(0, 12), closeTags: [], openTags: ["comment.block.js", "punctuation.definition.comment.js"]}
|
||||
{position: Point(0, 14), closeTags: ["punctuation.definition.comment.js"], openTags: []}
|
||||
{position: Point(1, 5), closeTags: [], openTags: ["punctuation.definition.comment.js"]}
|
||||
{position: Point(1, 7), closeTags: ["punctuation.definition.comment.js", "comment.block.js"], openTags: ["storage.type.var.js"]}
|
||||
{position: Point(1, 10), closeTags: ["storage.type.var.js"], openTags: []}
|
||||
{position: Point(1, 15), closeTags: [], openTags: ["keyword.operator.assignment.js"]}
|
||||
{position: Point(1, 16), closeTags: ["keyword.operator.assignment.js"], openTags: []}
|
||||
{position: Point(1, 17), closeTags: [], openTags: ["constant.numeric.js"]}
|
||||
{position: Point(1, 18), closeTags: ["constant.numeric.js"], openTags: []}
|
||||
])
|
||||
|
||||
expect(iterator.seek(Point(0, 1))).toEqual(["source.js", "storage.type.var.js"])
|
||||
expect(iterator.getPosition()).toEqual(Point(0, 3))
|
||||
expect(iterator.seek(Point(0, 8))).toEqual(["source.js"])
|
||||
expect(iterator.getPosition()).toEqual(Point(0, 8))
|
||||
expect(iterator.seek(Point(1, 0))).toEqual(["source.js", "comment.block.js"])
|
||||
expect(iterator.getPosition()).toEqual(Point(1, 5))
|
||||
expect(iterator.seek(Point(1, 18))).toEqual(["source.js", "constant.numeric.js"])
|
||||
expect(iterator.getPosition()).toEqual(Point(1, 18))
|
||||
|
||||
expect(iterator.seek(Point(2, 0))).toEqual(["source.js"])
|
||||
iterator.moveToSuccessor() # ensure we don't infinitely loop (regression test)
|
||||
|
||||
78
src/tokenized-buffer-iterator.coffee
Normal file
78
src/tokenized-buffer-iterator.coffee
Normal file
@@ -0,0 +1,78 @@
|
||||
{Point} = require 'text-buffer'
|
||||
|
||||
module.exports =
|
||||
class TokenizedBufferIterator
|
||||
constructor: (@tokenizedBuffer, @grammarRegistry) ->
|
||||
@openTags = null
|
||||
@closeTags = null
|
||||
|
||||
seek: (position) ->
|
||||
@openTags = []
|
||||
@closeTags = []
|
||||
@tagIndex = null
|
||||
|
||||
currentLine = @tokenizedBuffer.tokenizedLineForRow(position.row)
|
||||
containingTags = currentLine.openScopes.map (id) => @grammarRegistry.scopeForId(id)
|
||||
@currentTags = currentLine.tags
|
||||
currentColumn = 0
|
||||
for tag, index in @currentTags
|
||||
if tag >= 0
|
||||
if currentColumn >= position.column and @isAtTagBoundary()
|
||||
@tagIndex = index
|
||||
break
|
||||
else
|
||||
currentColumn += tag
|
||||
containingTags.pop() while @closeTags.shift()
|
||||
containingTags.push(tag) while tag = @openTags.shift()
|
||||
else
|
||||
scopeName = @grammarRegistry.scopeForId(tag)
|
||||
if tag % 2 is 0
|
||||
@closeTags.push(scopeName)
|
||||
else
|
||||
@openTags.push(scopeName)
|
||||
|
||||
@tagIndex ?= @currentTags.length
|
||||
@position = Point(position.row, currentColumn)
|
||||
containingTags
|
||||
|
||||
moveToSuccessor: ->
|
||||
if @tagIndex is @currentTags.length
|
||||
@position = Point(@position.row + 1, 0)
|
||||
@currentTags = @tokenizedBuffer.tokenizedLineForRow(@position.row)?.tags
|
||||
return false unless @currentTags?
|
||||
@tagIndex = 0
|
||||
else
|
||||
@position = Point(@position.row, @position.column + @currentTags[@tagIndex])
|
||||
@tagIndex++
|
||||
|
||||
@openTags = []
|
||||
@closeTags = []
|
||||
|
||||
loop
|
||||
tag = @currentTags[@tagIndex]
|
||||
if tag >= 0 or @tagIndex is @currentTags.length
|
||||
if @isAtTagBoundary()
|
||||
break
|
||||
else
|
||||
return @moveToSuccessor()
|
||||
else
|
||||
scopeName = @grammarRegistry.scopeForId(tag)
|
||||
if tag % 2 is 0
|
||||
@closeTags.push(scopeName)
|
||||
else
|
||||
@openTags.push(scopeName)
|
||||
@tagIndex++
|
||||
|
||||
true
|
||||
|
||||
getPosition: ->
|
||||
@position
|
||||
|
||||
getCloseTags: ->
|
||||
@closeTags.slice()
|
||||
|
||||
getOpenTags: ->
|
||||
@openTags.slice()
|
||||
|
||||
isAtTagBoundary: ->
|
||||
@closeTags.length > 0 or @openTags.length > 0
|
||||
@@ -7,6 +7,7 @@ TokenizedLine = require './tokenized-line'
|
||||
TokenIterator = require './token-iterator'
|
||||
Token = require './token'
|
||||
ScopeDescriptor = require './scope-descriptor'
|
||||
TokenizedBufferIterator = require './tokenized-buffer-iterator'
|
||||
|
||||
module.exports =
|
||||
class TokenizedBuffer extends Model
|
||||
@@ -58,6 +59,12 @@ class TokenizedBuffer extends Model
|
||||
destroyed: ->
|
||||
@disposables.dispose()
|
||||
|
||||
buildIterator: ->
|
||||
new TokenizedBufferIterator(this, @grammarRegistry)
|
||||
|
||||
getInvalidatedRanges: ->
|
||||
[@invalidatedRange]
|
||||
|
||||
serialize: ->
|
||||
state = {
|
||||
deserializer: 'TokenizedBuffer'
|
||||
@@ -274,6 +281,7 @@ class TokenizedBuffer extends Model
|
||||
[start, end] = @updateFoldableStatus(start, end + delta)
|
||||
end -= delta
|
||||
|
||||
@invalidatedRange = Range(start, end)
|
||||
event = {start, end, delta, bufferChange: e}
|
||||
@emitter.emit 'did-change', event
|
||||
|
||||
|
||||
@@ -47,143 +47,6 @@ class TokenizedLine
|
||||
@startBufferColumn ?= 0
|
||||
@bufferDelta = @text.length
|
||||
|
||||
@transformContent()
|
||||
@buildEndOfLineInvisibles() if @invisibles? and @lineEnding?
|
||||
|
||||
transformContent: ->
|
||||
text = ''
|
||||
bufferColumn = 0
|
||||
screenColumn = 0
|
||||
tokenIndex = 0
|
||||
tokenOffset = 0
|
||||
firstNonWhitespaceColumn = null
|
||||
lastNonWhitespaceColumn = null
|
||||
|
||||
substringStart = 0
|
||||
substringEnd = 0
|
||||
|
||||
while bufferColumn < @text.length
|
||||
# advance to next token if we've iterated over its length
|
||||
if tokenOffset is @tags[tokenIndex]
|
||||
tokenIndex++
|
||||
tokenOffset = 0
|
||||
|
||||
# advance to next token tag
|
||||
tokenIndex++ while @tags[tokenIndex] < 0
|
||||
|
||||
charCode = @text.charCodeAt(bufferColumn)
|
||||
|
||||
# split out unicode surrogate pairs
|
||||
if isPairedCharacter(@text, bufferColumn)
|
||||
prefix = tokenOffset
|
||||
suffix = @tags[tokenIndex] - tokenOffset - 2
|
||||
|
||||
i = tokenIndex
|
||||
@tags.splice(i, 1)
|
||||
@tags.splice(i++, 0, prefix) if prefix > 0
|
||||
@tags.splice(i++, 0, 2)
|
||||
@tags.splice(i, 0, suffix) if suffix > 0
|
||||
|
||||
firstNonWhitespaceColumn ?= screenColumn
|
||||
lastNonWhitespaceColumn = screenColumn + 1
|
||||
|
||||
substringEnd += 2
|
||||
screenColumn += 2
|
||||
bufferColumn += 2
|
||||
|
||||
tokenIndex++ if prefix > 0
|
||||
@specialTokens[tokenIndex] = PairedCharacter
|
||||
tokenIndex++
|
||||
tokenOffset = 0
|
||||
|
||||
# split out leading soft tabs
|
||||
else if charCode is SpaceCharCode
|
||||
if firstNonWhitespaceColumn?
|
||||
substringEnd += 1
|
||||
else
|
||||
if (screenColumn + 1) % @tabLength is 0
|
||||
suffix = @tags[tokenIndex] - @tabLength
|
||||
if suffix >= 0
|
||||
@specialTokens[tokenIndex] = SoftTab
|
||||
@tags.splice(tokenIndex, 1, @tabLength)
|
||||
@tags.splice(tokenIndex + 1, 0, suffix) if suffix > 0
|
||||
|
||||
if @invisibles?.space
|
||||
if substringEnd > substringStart
|
||||
text += @text.substring(substringStart, substringEnd)
|
||||
substringStart = substringEnd
|
||||
text += @invisibles.space
|
||||
substringStart += 1
|
||||
|
||||
substringEnd += 1
|
||||
|
||||
screenColumn++
|
||||
bufferColumn++
|
||||
tokenOffset++
|
||||
|
||||
# expand hard tabs to the next tab stop
|
||||
else if charCode is TabCharCode
|
||||
if substringEnd > substringStart
|
||||
text += @text.substring(substringStart, substringEnd)
|
||||
substringStart = substringEnd
|
||||
|
||||
tabLength = @tabLength - (screenColumn % @tabLength)
|
||||
if @invisibles?.tab
|
||||
text += @invisibles.tab
|
||||
text += getTabString(tabLength - 1) if tabLength > 1
|
||||
else
|
||||
text += getTabString(tabLength)
|
||||
|
||||
substringStart += 1
|
||||
substringEnd += 1
|
||||
|
||||
prefix = tokenOffset
|
||||
suffix = @tags[tokenIndex] - tokenOffset - 1
|
||||
|
||||
i = tokenIndex
|
||||
@tags.splice(i, 1)
|
||||
@tags.splice(i++, 0, prefix) if prefix > 0
|
||||
@tags.splice(i++, 0, tabLength)
|
||||
@tags.splice(i, 0, suffix) if suffix > 0
|
||||
|
||||
screenColumn += tabLength
|
||||
bufferColumn++
|
||||
|
||||
tokenIndex++ if prefix > 0
|
||||
@specialTokens[tokenIndex] = HardTab
|
||||
tokenIndex++
|
||||
tokenOffset = 0
|
||||
|
||||
# continue past any other character
|
||||
else
|
||||
firstNonWhitespaceColumn ?= screenColumn
|
||||
lastNonWhitespaceColumn = screenColumn
|
||||
|
||||
substringEnd += 1
|
||||
screenColumn++
|
||||
bufferColumn++
|
||||
tokenOffset++
|
||||
|
||||
if substringEnd > substringStart
|
||||
unless substringStart is 0 and substringEnd is @text.length
|
||||
text += @text.substring(substringStart, substringEnd)
|
||||
@text = text
|
||||
else
|
||||
@text = text
|
||||
|
||||
@firstNonWhitespaceIndex = firstNonWhitespaceColumn
|
||||
if lastNonWhitespaceColumn?
|
||||
if lastNonWhitespaceColumn + 1 < @text.length
|
||||
@firstTrailingWhitespaceIndex = lastNonWhitespaceColumn + 1
|
||||
if @invisibles?.space
|
||||
@text =
|
||||
@text.substring(0, @firstTrailingWhitespaceIndex) +
|
||||
@text.substring(@firstTrailingWhitespaceIndex)
|
||||
.replace(RepeatedSpaceRegex, @invisibles.space)
|
||||
else
|
||||
@lineIsWhitespaceOnly = true
|
||||
@firstTrailingWhitespaceIndex = 0
|
||||
|
||||
getTokenIterator: -> @tokenIterator.reset(this, arguments...)
|
||||
|
||||
Object.defineProperty @prototype, 'tokens', get: ->
|
||||
|
||||
Reference in New Issue
Block a user