Merge pull request #99 from github/async-tokenization

Tokenize asynchronously

src/app/cursor.coffee

@@ -107,7 +107,6 @@ class Cursor
   moveToFirstCharacterOfLine: ->
     position = @getBufferPosition()
     range = @getCurrentLineBufferRange()
-    console.log range.inspect()
     newPosition = null
     @editSession.scanInRange /^\s*/, range, (match, matchRange) =>
       newPosition = matchRange.end

src/app/display-buffer.coffee

@@ -27,6 +27,8 @@ class DisplayBuffer
     @buildLineMap()
     @tokenizedBuffer.on 'change', (e) => @handleTokenizedBufferChange(e)
 
+  setVisible: (visible) -> @tokenizedBuffer.setVisible(visible)
+
   buildLineMap: ->
     @lineMap = new LineMap
     @lineMap.insertAtScreenRow 0, @buildLinesForBufferRows(0, @buffer.getLastRow())

src/app/edit-session.coffee

@@ -91,6 +91,8 @@ class EditSession
       @scrollLeft == other.getScrollLeft() and
       @getCursorScreenPosition().isEqual(other.getCursorScreenPosition())
 
+  setVisible: (visible) -> @displayBuffer.setVisible(visible)
+
   setScrollTop: (@scrollTop) ->
   getScrollTop: -> @scrollTop
 

src/app/editor.coffee

@@ -425,6 +425,7 @@ class Editor extends View
       @activeEditSession.off()
 
     @activeEditSession = @editSessions[index]
+    @activeEditSession.setVisible(true)
 
     @activeEditSession.on "buffer-contents-change-on-disk", =>
       @showBufferConflictAlert(@activeEditSession)

src/app/tokenized-buffer.coffee

@@ -14,63 +14,126 @@ class TokenizedBuffer
   buffer: null
   aceAdaptor: null
   screenLines: null
+  chunkSize: 50
+  invalidRows: null
+  visible: false
 
   constructor: (@buffer, { @languageMode, @tabLength }) ->
     @tabLength ?= 2
     @id = @constructor.idCounter++
-    @screenLines = @buildScreenLinesForRows(0, @buffer.getLastRow())
+    @screenLines = @buildPlaceholderScreenLinesForRows(0, @buffer.getLastRow())
+    @invalidRows = []
+    @invalidateRow(0)
     @buffer.on "change.tokenized-buffer#{@id}", (e) => @handleBufferChange(e)
 
+  setVisible: (@visible) ->
+    @tokenizeInBackground() if @visible
+
+  getTabLength: ->
+    @tabLength
+
+  setTabLength: (@tabLength) ->
+    lastRow = @buffer.getLastRow()
+    @screenLines = @buildPlaceholderScreenLinesForRows(0, lastRow)
+    @invalidateRow(0)
+    @trigger "change", { start: 0, end: lastRow, delta: 0 }
+
+  tokenizeInBackground: ->
+    return if not @visible or @pendingChunk
+    @pendingChunk = true
+    _.defer =>
+      @pendingChunk = false
+      @tokenizeNextChunk()
+
+  tokenizeNextChunk: ->
+    rowsRemaining = @chunkSize
+
+    while @firstInvalidRow()? and rowsRemaining > 0
+      invalidRow = @invalidRows.shift()
+      lastRow = @getLastRow()
+      continue if invalidRow > lastRow
+
+      row = invalidRow
+      loop
+        previousStack = @stackForRow(row)
+        @screenLines[row] = @buildTokenizedScreenLineForRow(row, @stackForRow(row - 1))
+        if --rowsRemaining == 0
+          filledRegion = false
+          break
+        if row == lastRow or _.isEqual(@stackForRow(row), previousStack)
+          filledRegion = true
+          break
+        row++
+
+      @validateRow(row)
+      @invalidateRow(row + 1) unless filledRegion
+      @trigger "change", { start: invalidRow, end: row, delta: 0 }
+
+    @tokenizeInBackground() if @firstInvalidRow()?
+
+  firstInvalidRow: ->
+    @invalidRows[0]
+
+  validateRow: (row) ->
+    @invalidRows.shift() while @invalidRows[0] <= row
+
+  invalidateRow: (row) ->
+    @invalidRows.push(row)
+    @invalidRows.sort (a, b) -> a - b
+    @tokenizeInBackground()
+
+  updateInvalidRows: (start, end, delta) ->
+    @invalidRows = @invalidRows.map (row) ->
+      if row < start
+        row
+      else if start <= row <= end
+        end + delta + 1
+      else if row > end
+        row + delta
+
   handleBufferChange: (e) ->
     {oldRange, newRange} = e
     start = oldRange.start.row
     end = oldRange.end.row
     delta = newRange.end.row - oldRange.end.row
 
+    @updateInvalidRows(start, end, delta)
+
     previousStack = @stackForRow(end) # used in spill detection below
 
     stack = @stackForRow(start - 1)
-    @screenLines[start..end] = @buildScreenLinesForRows(start, end + delta, stack)
+    if stack? or start == 0
+      @screenLines[start..end] = @buildTokenizedScreenLinesForRows(start, end + delta, stack)
+    else
+      @screenLines[start..end] = @buildPlaceholderScreenLinesForRows(start, end + delta, stack)
 
-    # spill detection
-    # compare scanner state of last re-highlighted line with its previous state.
-    # if it differs, re-tokenize the next line with the new state and repeat for
-    # each line until the line's new state matches the previous state. this covers
-    # cases like inserting a /* needing to comment out lines below until we see a */
-    for row in [(end + delta)...@buffer.getLastRow()]
-      break if _.isEqual(@stackForRow(row), previousStack)
-      nextRow = row + 1
-      previousStack = @stackForRow(nextRow)
-      @screenLines[nextRow] = @buildScreenLineForRow(nextRow, @stackForRow(row))
-
-    # if highlighting spilled beyond the bounds of the textual change, update the
-    # end of the affected range to reflect the larger area of highlighting
-    end = Math.max(end, nextRow - delta) if nextRow
+    unless _.isEqual(@stackForRow(end + delta), previousStack)
+      @invalidateRow(end + delta + 1)
 
     @trigger "change", { start, end, delta, bufferChange: e }
 
-  getTabLength: ->
-    @tabLength
+  buildPlaceholderScreenLinesForRows: (startRow, endRow) ->
+    @buildPlaceholderScreenLineForRow(row) for row in [startRow..endRow]
 
-  setTabLength: (@tabLength) ->
-    lastRow = @buffer.getLastRow()
-    @screenLines = @buildScreenLinesForRows(0, lastRow)
-    @trigger "change", { start: 0, end: lastRow, delta: 0 }
+  buildPlaceholderScreenLineForRow: (row) ->
+    line = @buffer.lineForRow(row)
+    tokens = [new Token(value: line, scopes: [@languageMode.grammar.scopeName])]
+    new ScreenLine({tokens, @tabLength})
 
-  buildScreenLinesForRows: (startRow, endRow, startingStack) ->
+  buildTokenizedScreenLinesForRows: (startRow, endRow, startingStack) ->
     ruleStack = startingStack
     for row in [startRow..endRow]
-      screenLine = @buildScreenLineForRow(row, ruleStack)
+      screenLine = @buildTokenizedScreenLineForRow(row, ruleStack)
       ruleStack = screenLine.ruleStack
       screenLine
 
-  buildScreenLineForRow: (row, ruleStack) ->
+  buildTokenizedScreenLineForRow: (row, ruleStack) ->
     line = @buffer.lineForRow(row)
     { tokens, ruleStack } = @languageMode.tokenizeLine(line, ruleStack)
     new ScreenLine({tokens, ruleStack, @tabLength})
 
   lineForScreenRow: (row) ->
-    @screenLines[row]
+    @linesForScreenRows(row, row)[0]
 
   linesForScreenRows: (startRow, endRow) ->
     @screenLines[startRow..endRow]
@@ -146,6 +209,9 @@ class TokenizedBuffer
             stop()
     position
 
+  getLastRow: ->
+    @buffer.getLastRow()
+
   logLines: (start=0, end=@buffer.getLastRow()) ->
     for row in [start..end]
       line = @lineForScreenRow(row).text