Make TokenizedBuffer conform to text decoration layer interface

src/tokenized-buffer-iterator.coffee

@@ -0,0 +1,78 @@
+{Point} = require 'text-buffer'
+
+module.exports =
+class TokenizedBufferIterator
+  constructor: (@tokenizedBuffer, @grammarRegistry) ->
+    @openTags = null
+    @closeTags = null
+
+  seek: (position) ->
+    @openTags = []
+    @closeTags = []
+    @tagIndex = null
+
+    currentLine = @tokenizedBuffer.tokenizedLineForRow(position.row)
+    containingTags = currentLine.openScopes.map (id) => @grammarRegistry.scopeForId(id)
+    @currentTags = currentLine.tags
+    currentColumn = 0
+    for tag, index in @currentTags
+      if tag >= 0
+        if currentColumn >= position.column and @isAtTagBoundary()
+          @tagIndex = index
+          break
+        else
+          currentColumn += tag
+          containingTags.pop() while @closeTags.shift()
+          containingTags.push(tag) while tag = @openTags.shift()
+      else
+        scopeName = @grammarRegistry.scopeForId(tag)
+        if tag % 2 is 0
+          @closeTags.push(scopeName)
+        else
+          @openTags.push(scopeName)
+
+    @tagIndex ?= @currentTags.length
+    @position = Point(position.row, currentColumn)
+    containingTags
+
+  moveToSuccessor: ->
+    if @tagIndex is @currentTags.length
+      @position = Point(@position.row + 1, 0)
+      @currentTags = @tokenizedBuffer.tokenizedLineForRow(@position.row)?.tags
+      return false unless @currentTags?
+      @tagIndex = 0
+    else
+      @position = Point(@position.row, @position.column + @currentTags[@tagIndex])
+      @tagIndex++
+
+    @openTags = []
+    @closeTags = []
+
+    loop
+      tag = @currentTags[@tagIndex]
+      if tag >= 0 or @tagIndex is @currentTags.length
+        if @isAtTagBoundary()
+          break
+        else
+          return @moveToSuccessor()
+      else
+        scopeName = @grammarRegistry.scopeForId(tag)
+        if tag % 2 is 0
+          @closeTags.push(scopeName)
+        else
+          @openTags.push(scopeName)
+      @tagIndex++
+
+    true
+
+  getPosition: ->
+    @position
+
+  getCloseTags: ->
+    @closeTags.slice()
+
+  getOpenTags: ->
+    @openTags.slice()
+
+  isAtTagBoundary: ->
+    @closeTags.length > 0 or @openTags.length > 0

src/tokenized-buffer.coffee

@@ -7,6 +7,7 @@ TokenizedLine = require './tokenized-line'
 TokenIterator = require './token-iterator'
 Token = require './token'
 ScopeDescriptor = require './scope-descriptor'
+TokenizedBufferIterator = require './tokenized-buffer-iterator'
 
 module.exports =
 class TokenizedBuffer extends Model
@@ -58,6 +59,12 @@ class TokenizedBuffer extends Model
   destroyed: ->
     @disposables.dispose()
 
+  buildIterator: ->
+    new TokenizedBufferIterator(this, @grammarRegistry)
+
+  getInvalidatedRanges: ->
+    [@invalidatedRange]
+
   serialize: ->
     state = {
       deserializer: 'TokenizedBuffer'
@@ -274,6 +281,7 @@ class TokenizedBuffer extends Model
     [start, end] = @updateFoldableStatus(start, end + delta)
     end -= delta
 
+    @invalidatedRange = Range(start, end)
     event = {start, end, delta, bufferChange: e}
     @emitter.emit 'did-change', event
 

src/tokenized-line.coffee

@@ -47,143 +47,6 @@ class TokenizedLine
     @startBufferColumn ?= 0
     @bufferDelta = @text.length
 
-    @transformContent()
-    @buildEndOfLineInvisibles() if @invisibles? and @lineEnding?
-
-  transformContent: ->
-    text = ''
-    bufferColumn = 0
-    screenColumn = 0
-    tokenIndex = 0
-    tokenOffset = 0
-    firstNonWhitespaceColumn = null
-    lastNonWhitespaceColumn = null
-
-    substringStart = 0
-    substringEnd = 0
-
-    while bufferColumn < @text.length
-      # advance to next token if we've iterated over its length
-      if tokenOffset is @tags[tokenIndex]
-        tokenIndex++
-        tokenOffset = 0
-
-      # advance to next token tag
-      tokenIndex++ while @tags[tokenIndex] < 0
-
-      charCode = @text.charCodeAt(bufferColumn)
-
-      # split out unicode surrogate pairs
-      if isPairedCharacter(@text, bufferColumn)
-        prefix = tokenOffset
-        suffix = @tags[tokenIndex] - tokenOffset - 2
-
-        i = tokenIndex
-        @tags.splice(i, 1)
-        @tags.splice(i++, 0, prefix) if prefix > 0
-        @tags.splice(i++, 0, 2)
-        @tags.splice(i, 0, suffix) if suffix > 0
-
-        firstNonWhitespaceColumn ?= screenColumn
-        lastNonWhitespaceColumn = screenColumn + 1
-
-        substringEnd += 2
-        screenColumn += 2
-        bufferColumn += 2
-
-        tokenIndex++ if prefix > 0
-        @specialTokens[tokenIndex] = PairedCharacter
-        tokenIndex++
-        tokenOffset = 0
-
-      # split out leading soft tabs
-      else if charCode is SpaceCharCode
-        if firstNonWhitespaceColumn?
-          substringEnd += 1
-        else
-          if (screenColumn + 1) % @tabLength is 0
-            suffix = @tags[tokenIndex] - @tabLength
-            if suffix >= 0
-              @specialTokens[tokenIndex] = SoftTab
-              @tags.splice(tokenIndex, 1, @tabLength)
-              @tags.splice(tokenIndex + 1, 0, suffix) if suffix > 0
-
-          if @invisibles?.space
-            if substringEnd > substringStart
-              text += @text.substring(substringStart, substringEnd)
-            substringStart = substringEnd
-            text += @invisibles.space
-            substringStart += 1
-
-          substringEnd += 1
-
-        screenColumn++
-        bufferColumn++
-        tokenOffset++
-
-      # expand hard tabs to the next tab stop
-      else if charCode is TabCharCode
-        if substringEnd > substringStart
-          text += @text.substring(substringStart, substringEnd)
-        substringStart = substringEnd
-
-        tabLength = @tabLength - (screenColumn % @tabLength)
-        if @invisibles?.tab
-          text += @invisibles.tab
-          text += getTabString(tabLength - 1) if tabLength > 1
-        else
-          text += getTabString(tabLength)
-
-        substringStart += 1
-        substringEnd += 1
-
-        prefix = tokenOffset
-        suffix = @tags[tokenIndex] - tokenOffset - 1
-
-        i = tokenIndex
-        @tags.splice(i, 1)
-        @tags.splice(i++, 0, prefix) if prefix > 0
-        @tags.splice(i++, 0, tabLength)
-        @tags.splice(i, 0, suffix) if suffix > 0
-
-        screenColumn += tabLength
-        bufferColumn++
-
-        tokenIndex++ if prefix > 0
-        @specialTokens[tokenIndex] = HardTab
-        tokenIndex++
-        tokenOffset = 0
-
-      # continue past any other character
-      else
-        firstNonWhitespaceColumn ?= screenColumn
-        lastNonWhitespaceColumn = screenColumn
-
-        substringEnd += 1
-        screenColumn++
-        bufferColumn++
-        tokenOffset++
-
-    if substringEnd > substringStart
-      unless substringStart is 0 and substringEnd is @text.length
-        text += @text.substring(substringStart, substringEnd)
-        @text = text
-    else
-      @text = text
-
-    @firstNonWhitespaceIndex = firstNonWhitespaceColumn
-    if lastNonWhitespaceColumn?
-      if lastNonWhitespaceColumn + 1 < @text.length
-        @firstTrailingWhitespaceIndex = lastNonWhitespaceColumn + 1
-        if @invisibles?.space
-          @text =
-            @text.substring(0, @firstTrailingWhitespaceIndex) +
-              @text.substring(@firstTrailingWhitespaceIndex)
-                .replace(RepeatedSpaceRegex, @invisibles.space)
-    else
-      @lineIsWhitespaceOnly = true
-      @firstTrailingWhitespaceIndex = 0
-
   getTokenIterator: -> @tokenIterator.reset(this, arguments...)
 
   Object.defineProperty @prototype, 'tokens', get: ->