Merge pull request #99 from github/async-tokenization

Tokenize asynchronously

spec/app/display-buffer-spec.coffee

@@ -565,4 +565,3 @@ describe "DisplayBuffer", ->
   describe ".maxLineLength()", ->
     it "returns the length of the longest screen line", ->
       expect(displayBuffer.maxLineLength()).toBe 65
-

spec/app/tokenized-buffer-spec.coffee

@@ -5,166 +5,302 @@ Range = require 'range'
 _ = require 'underscore'
 
 describe "TokenizedBuffer", ->
-  [editSession, tokenizedBuffer, buffer] = []
+  [editSession, tokenizedBuffer, buffer, changeHandler] = []
 
   beforeEach ->
-    editSession = fixturesProject.buildEditSessionForPath('sample.js', autoIndent: false)
-    buffer = editSession.buffer
-    tokenizedBuffer = editSession.displayBuffer.tokenizedBuffer
+    # enable async tokenization
+    TokenizedBuffer.prototype.chunkSize = 5
+    jasmine.unspy(TokenizedBuffer.prototype, 'tokenizeInBackground')
 
-  afterEach ->
-    editSession.destroy()
+  fullyTokenize = (tokenizedBuffer) ->
+    advanceClock() while tokenizedBuffer.firstInvalidRow()?
+    changeHandler.reset()
 
-  describe ".findOpeningBracket(closingBufferPosition)", ->
-    it "returns the position of the matching bracket, skipping any nested brackets", ->
-      expect(tokenizedBuffer.findOpeningBracket([9, 2])).toEqual [1, 29]
+  describe "when the buffer contains soft-tabs", ->
+    beforeEach ->
+      editSession = fixturesProject.buildEditSessionForPath('sample.js', autoIndent: false)
+      buffer = editSession.buffer
+      tokenizedBuffer = editSession.displayBuffer.tokenizedBuffer
+      editSession.setVisible(true)
+      changeHandler = jasmine.createSpy('changeHandler')
+      tokenizedBuffer.on "change", changeHandler
 
-  describe ".findClosingBracket(startBufferPosition)", ->
-    it "returns the position of the matching bracket, skipping any nested brackets", ->
-      expect(tokenizedBuffer.findClosingBracket([1, 29])).toEqual [9, 2]
+    afterEach ->
+      editSession.destroy()
 
-  describe "tokenization", ->
-    it "tokenizes all the lines in the buffer on construction", ->
-      expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.modifier.js'])
-      expect(tokenizedBuffer.lineForScreenRow(11).tokens[1]).toEqual(value: 'return', scopes: ['source.js', 'keyword.control.js'])
+    describe "on construction", ->
+      it "initially creates un-tokenized screen lines, then tokenizes lines chunk at a time in the background", ->
+        line0 = tokenizedBuffer.lineForScreenRow(0)
+        expect(line0.tokens.length).toBe 1
+        expect(line0.tokens[0]).toEqual(value: line0.text, scopes: ['source.js'])
 
-    describe "when the buffer changes", ->
-      changeHandler = null
+        line11 = tokenizedBuffer.lineForScreenRow(11)
+        expect(line11.tokens.length).toBe 2
+        expect(line11.tokens[0]).toEqual(value: "  ", scopes: ['source.js'], isAtomic: true)
+        expect(line11.tokens[1]).toEqual(value: "return sort(Array.apply(this, arguments));", scopes: ['source.js'])
 
+        # background tokenization has not begun
+        expect(tokenizedBuffer.lineForScreenRow(0).ruleStack).toBeUndefined()
+
+        # tokenize chunk 1
+        advanceClock()
+        expect(tokenizedBuffer.lineForScreenRow(0).ruleStack?).toBeTruthy()
+        expect(tokenizedBuffer.lineForScreenRow(4).ruleStack?).toBeTruthy()
+        expect(tokenizedBuffer.lineForScreenRow(5).ruleStack?).toBeFalsy()
+        expect(changeHandler).toHaveBeenCalledWith(start: 0, end: 4, delta: 0)
+        changeHandler.reset()
+
+        # tokenize chunk 2
+        advanceClock()
+        expect(tokenizedBuffer.lineForScreenRow(5).ruleStack?).toBeTruthy()
+        expect(tokenizedBuffer.lineForScreenRow(9).ruleStack?).toBeTruthy()
+        expect(tokenizedBuffer.lineForScreenRow(10).ruleStack?).toBeFalsy()
+        expect(changeHandler).toHaveBeenCalledWith(start: 5, end: 9, delta: 0)
+        changeHandler.reset()
+
+        # tokenize last chunk
+        advanceClock()
+        expect(tokenizedBuffer.lineForScreenRow(10).ruleStack?).toBeTruthy()
+        expect(tokenizedBuffer.lineForScreenRow(12).ruleStack?).toBeTruthy()
+        expect(changeHandler).toHaveBeenCalledWith(start: 10, end: 12, delta: 0)
+
+    describe "when the buffer is partially tokenized", ->
       beforeEach ->
-        changeHandler = jasmine.createSpy('changeHandler')
-        tokenizedBuffer.on "change", changeHandler
+        # tokenize chunk 1 only
+        advanceClock()
+        changeHandler.reset()
 
-      describe "when lines are updated, but none are added or removed", ->
-        it "updates tokens for each of the changed lines", ->
-          range = new Range([0, 0], [2, 0])
-          buffer.change(range, "foo()\n7\n")
+      describe "when there is a buffer change inside the tokenized region", ->
+        describe "when lines are added", ->
+          it "pushes the invalid rows down", ->
+            expect(tokenizedBuffer.firstInvalidRow()).toBe 5
+            buffer.insert([1, 0], '\n\n')
+            changeHandler.reset()
 
-          expect(tokenizedBuffer.lineForScreenRow(0).tokens[1]).toEqual(value: '(', scopes: ['source.js', 'meta.brace.round.js'])
-          expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: '7', scopes: ['source.js', 'constant.numeric.js'])
-          # line 2 is unchanged
-          expect(tokenizedBuffer.lineForScreenRow(2).tokens[2]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js'])
+            expect(tokenizedBuffer.firstInvalidRow()).toBe 7
+            advanceClock()
+            expect(changeHandler).toHaveBeenCalledWith(start: 7, end: 11, delta: 0)
 
-          expect(changeHandler).toHaveBeenCalled()
-          [event] = changeHandler.argsForCall[0]
-          delete event.bufferChange
-          expect(event).toEqual(start: 0, end: 2, delta: 0)
+        describe "when lines are removed", ->
+          it "pulls the invalid rows up", ->
+            expect(tokenizedBuffer.firstInvalidRow()).toBe 5
+            buffer.delete([[1, 0], [3, 0]])
+            changeHandler.reset()
 
-        it "updates tokens for lines beyond the changed lines if needed", ->
-          buffer.insert([5, 30], '/* */')
+            expect(tokenizedBuffer.firstInvalidRow()).toBe 3
+            advanceClock()
+            expect(changeHandler).toHaveBeenCalledWith(start: 3, end: 7, delta: 0)
+
+        describe "when the change invalidates all the lines before the current invalid region", ->
+          it "retokenizes the invalidated lines and continues into the valid region", ->
+            expect(tokenizedBuffer.firstInvalidRow()).toBe 5
+            buffer.insert([2, 0], '/*')
+            changeHandler.reset()
+            expect(tokenizedBuffer.firstInvalidRow()).toBe 3
+
+            advanceClock()
+            expect(changeHandler).toHaveBeenCalledWith(start: 3, end: 7, delta: 0)
+            expect(tokenizedBuffer.firstInvalidRow()).toBe 8
+
+      describe "when there is a buffer change surrounding an invalid row", ->
+        it "pushes the invalid row to the end of the change", ->
+          buffer.change([[4, 0], [6, 0]], "\n\n\n")
           changeHandler.reset()
 
-          buffer.insert([2, 0], '/*')
-          expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
-          expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
-          expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
+          expect(tokenizedBuffer.firstInvalidRow()).toBe 8
+          advanceClock()
 
-          expect(changeHandler).toHaveBeenCalled()
-          [event] = changeHandler.argsForCall[0]
-          delete event.bufferChange
-          expect(event).toEqual(start: 2, end: 5, delta: 0)
+      describe "when there is a buffer change inside an invalid region", ->
+        it "does not attempt to tokenize the lines in the change, and preserves the existing invalid row", ->
+          expect(tokenizedBuffer.firstInvalidRow()).toBe 5
+          buffer.change([[6, 0], [7, 0]], "\n\n\n")
 
-        it "resumes highlighting with the state of the previous line", ->
-          buffer.insert([0, 0], '/*')
-          buffer.insert([5, 0], '*/')
+          expect(tokenizedBuffer.lineForScreenRow(6).ruleStack?).toBeFalsy()
+          expect(tokenizedBuffer.lineForScreenRow(7).ruleStack?).toBeFalsy()
 
-          buffer.insert([1, 0], 'var ')
-          expect(tokenizedBuffer.lineForScreenRow(1).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
-
-      describe "when lines are both updated and removed", ->
-        it "updates tokens to reflect the removed lines", ->
-          range = new Range([1, 0], [3, 0])
-          buffer.change(range, "foo()")
-
-          # previous line 0 remains
-          expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.modifier.js'])
-
-          # previous line 3 should be combined with input to form line 1
-          expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js'])
-          expect(tokenizedBuffer.lineForScreenRow(1).tokens[6]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
-
-          # lines below deleted regions should be shifted upward
-          expect(tokenizedBuffer.lineForScreenRow(2).tokens[2]).toEqual(value: 'while', scopes: ['source.js', 'keyword.control.js'])
-          expect(tokenizedBuffer.lineForScreenRow(3).tokens[4]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
-          expect(tokenizedBuffer.lineForScreenRow(4).tokens[4]).toEqual(value: '<', scopes: ['source.js', 'keyword.operator.js'])
-
-          expect(changeHandler).toHaveBeenCalled()
-          [event] = changeHandler.argsForCall[0]
-          delete event.bufferChange
-          expect(event).toEqual(start: 1, end: 3, delta: -2)
-
-        it "updates tokens for lines beyond the changed lines if needed", ->
-          buffer.insert([5, 30], '/* */')
           changeHandler.reset()
+          expect(tokenizedBuffer.firstInvalidRow()).toBe 5
 
-          buffer.change(new Range([2, 0], [3, 0]), '/*')
-          expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js']
-          expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
-          expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
-
-          expect(changeHandler).toHaveBeenCalled()
-          [event] = changeHandler.argsForCall[0]
-          delete event.bufferChange
-          expect(event).toEqual(start: 2, end: 5, delta: -1)
-
-      describe "when lines are both updated and inserted", ->
-        it "updates tokens to reflect the inserted lines", ->
-          range = new Range([1, 0], [2, 0])
-          buffer.change(range, "foo()\nbar()\nbaz()\nquux()")
-
-          # previous line 0 remains
-          expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual( value: 'var', scopes: ['source.js', 'storage.modifier.js'])
-
-          # 3 new lines inserted
-          expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js'])
-          expect(tokenizedBuffer.lineForScreenRow(2).tokens[0]).toEqual(value: 'bar', scopes: ['source.js'])
-          expect(tokenizedBuffer.lineForScreenRow(3).tokens[0]).toEqual(value: 'baz', scopes: ['source.js'])
-
-          # previous line 2 is joined with quux() on line 4
-          expect(tokenizedBuffer.lineForScreenRow(4).tokens[0]).toEqual(value: 'quux', scopes: ['source.js'])
-          expect(tokenizedBuffer.lineForScreenRow(4).tokens[4]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js'])
-
-          # previous line 3 is pushed down to become line 5
-          expect(tokenizedBuffer.lineForScreenRow(5).tokens[4]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
-
-          expect(changeHandler).toHaveBeenCalled()
-          [event] = changeHandler.argsForCall[0]
-          delete event.bufferChange
-          expect(event).toEqual(start: 1, end: 2, delta: 2)
-
-        it "updates tokens for lines beyond the changed lines if needed", ->
-          buffer.insert([5, 30], '/* */')
-          changeHandler.reset()
-
-          buffer.insert([2, 0], '/*\nabcde\nabcder')
-          expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js']
-          expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
-          expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
-          expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
-          expect(tokenizedBuffer.lineForScreenRow(6).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
-          expect(tokenizedBuffer.lineForScreenRow(7).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
-          expect(tokenizedBuffer.lineForScreenRow(8).tokens[0].scopes).not.toBe ['source.js', 'comment.block.js']
-
-          expect(changeHandler).toHaveBeenCalled()
-          [event] = changeHandler.argsForCall[0]
-          delete event.bufferChange
-          expect(event).toEqual(start: 2, end: 5, delta: 2)
-
-
-    describe "when the buffer contains tab characters", ->
-      editSession2 = null
-
+    describe "when the buffer is fully tokenized", ->
       beforeEach ->
-        tabLength = 2
-        editSession2 = fixturesProject.buildEditSessionForPath('sample-with-tabs.coffee', { tabLength })
-        buffer = editSession2.buffer
-        tokenizedBuffer = editSession2.displayBuffer.tokenizedBuffer
+        fullyTokenize(tokenizedBuffer)
 
-      afterEach ->
-        editSession2.destroy()
+      describe "when there is a buffer change that is smaller than the chunk size", ->
+        describe "when lines are updated, but none are added or removed", ->
+          it "updates tokens to reflect the change", ->
+            buffer.change([[0, 0], [2, 0]], "foo()\n7\n")
 
-      it "always renders each tab as its own atomic token with a value of size tabLength", ->
-        tabAsSpaces = _.multiplyString(' ', editSession2.getTabLength())
+            expect(tokenizedBuffer.lineForScreenRow(0).tokens[1]).toEqual(value: '(', scopes: ['source.js', 'meta.brace.round.js'])
+            expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: '7', scopes: ['source.js', 'constant.numeric.js'])
+            # line 2 is unchanged
+            expect(tokenizedBuffer.lineForScreenRow(2).tokens[2]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js'])
+
+            expect(changeHandler).toHaveBeenCalled()
+            [event] = changeHandler.argsForCall[0]
+            delete event.bufferChange
+            expect(event).toEqual(start: 0, end: 2, delta: 0)
+
+          describe "when the change invalidates the tokenization of subsequent lines", ->
+            it "schedules the invalidated lines to be tokenized in the background", ->
+              buffer.insert([5, 30], '/* */')
+              changeHandler.reset()
+              buffer.insert([2, 0], '/*')
+              expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js']
+              expect(changeHandler).toHaveBeenCalled()
+              [event] = changeHandler.argsForCall[0]
+              delete event.bufferChange
+              expect(event).toEqual(start: 2, end: 2, delta: 0)
+              changeHandler.reset()
+
+              advanceClock()
+              expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
+              expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
+              expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
+              expect(changeHandler).toHaveBeenCalled()
+              [event] = changeHandler.argsForCall[0]
+              delete event.bufferChange
+              expect(event).toEqual(start: 3, end: 5, delta: 0)
+
+          it "resumes highlighting with the state of the previous line", ->
+            buffer.insert([0, 0], '/*')
+            buffer.insert([5, 0], '*/')
+
+            buffer.insert([1, 0], 'var ')
+            expect(tokenizedBuffer.lineForScreenRow(1).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
+
+        describe "when lines are both updated and removed", ->
+          it "updates tokens to reflect the change", ->
+            buffer.change([[1, 0], [3, 0]], "foo()")
+
+            # previous line 0 remains
+            expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.modifier.js'])
+
+            # previous line 3 should be combined with input to form line 1
+            expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js'])
+            expect(tokenizedBuffer.lineForScreenRow(1).tokens[6]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
+
+            # lines below deleted regions should be shifted upward
+            expect(tokenizedBuffer.lineForScreenRow(2).tokens[2]).toEqual(value: 'while', scopes: ['source.js', 'keyword.control.js'])
+            expect(tokenizedBuffer.lineForScreenRow(3).tokens[4]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
+            expect(tokenizedBuffer.lineForScreenRow(4).tokens[4]).toEqual(value: '<', scopes: ['source.js', 'keyword.operator.js'])
+
+            expect(changeHandler).toHaveBeenCalled()
+            [event] = changeHandler.argsForCall[0]
+            delete event.bufferChange
+            expect(event).toEqual(start: 1, end: 3, delta: -2)
+
+        describe "when the change invalidates the tokenization of subsequent lines", ->
+          it "schedules the invalidated lines to be tokenized in the background", ->
+            buffer.insert([5, 30], '/* */')
+            changeHandler.reset()
+
+            buffer.change([[2, 0], [3, 0]], '/*')
+            expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js']
+            expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js']
+            expect(changeHandler).toHaveBeenCalled()
+            [event] = changeHandler.argsForCall[0]
+            delete event.bufferChange
+            expect(event).toEqual(start: 2, end: 3, delta: -1)
+            changeHandler.reset()
+
+            advanceClock()
+            expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
+            expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
+            expect(changeHandler).toHaveBeenCalled()
+            [event] = changeHandler.argsForCall[0]
+            delete event.bufferChange
+            expect(event).toEqual(start: 3, end: 4, delta: 0)
+
+        describe "when lines are both updated and inserted", ->
+          it "updates tokens to reflect the change", ->
+            buffer.change([[1, 0], [2, 0]], "foo()\nbar()\nbaz()\nquux()")
+
+            # previous line 0 remains
+            expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual( value: 'var', scopes: ['source.js', 'storage.modifier.js'])
+
+            # 3 new lines inserted
+            expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js'])
+            expect(tokenizedBuffer.lineForScreenRow(2).tokens[0]).toEqual(value: 'bar', scopes: ['source.js'])
+            expect(tokenizedBuffer.lineForScreenRow(3).tokens[0]).toEqual(value: 'baz', scopes: ['source.js'])
+
+            # previous line 2 is joined with quux() on line 4
+            expect(tokenizedBuffer.lineForScreenRow(4).tokens[0]).toEqual(value: 'quux', scopes: ['source.js'])
+            expect(tokenizedBuffer.lineForScreenRow(4).tokens[4]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js'])
+
+            # previous line 3 is pushed down to become line 5
+            expect(tokenizedBuffer.lineForScreenRow(5).tokens[4]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
+
+            expect(changeHandler).toHaveBeenCalled()
+            [event] = changeHandler.argsForCall[0]
+            delete event.bufferChange
+            expect(event).toEqual(start: 1, end: 2, delta: 2)
+
+        describe "when the change invalidates the tokenization of subsequent lines", ->
+          it "schedules the invalidated lines to be tokenized in the background", ->
+            buffer.insert([5, 30], '/* */')
+            changeHandler.reset()
+
+            buffer.insert([2, 0], '/*\nabcde\nabcder')
+            expect(changeHandler).toHaveBeenCalled()
+            [event] = changeHandler.argsForCall[0]
+            delete event.bufferChange
+            expect(event).toEqual(start: 2, end: 2, delta: 2)
+            expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js']
+            expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
+            expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
+            expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js']
+            changeHandler.reset()
+
+            advanceClock() # tokenize invalidated lines in background
+            expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
+            expect(tokenizedBuffer.lineForScreenRow(6).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
+            expect(tokenizedBuffer.lineForScreenRow(7).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
+            expect(tokenizedBuffer.lineForScreenRow(8).tokens[0].scopes).not.toBe ['source.js', 'comment.block.js']
+
+            expect(changeHandler).toHaveBeenCalled()
+            [event] = changeHandler.argsForCall[0]
+            delete event.bufferChange
+            expect(event).toEqual(start: 5, end: 7, delta: 0)
+
+      describe ".findOpeningBracket(closingBufferPosition)", ->
+        it "returns the position of the matching bracket, skipping any nested brackets", ->
+          expect(tokenizedBuffer.findOpeningBracket([9, 2])).toEqual [1, 29]
+
+      describe ".findClosingBracket(startBufferPosition)", ->
+        it "returns the position of the matching bracket, skipping any nested brackets", ->
+          expect(tokenizedBuffer.findClosingBracket([1, 29])).toEqual [9, 2]
+
+      it "tokenizes leading whitespace based on the new tab length", ->
+        expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].isAtomic).toBeTruthy()
+        expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].value).toBe "  "
+        expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].isAtomic).toBeTruthy()
+        expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].value).toBe "  "
+
+        tokenizedBuffer.setTabLength(4)
+        fullyTokenize(tokenizedBuffer)
+
+        expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].isAtomic).toBeTruthy()
+        expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].value).toBe "    "
+        expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].isAtomic).toBeFalsy()
+        expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].value).toBe "  current "
+
+  describe "when the buffer contains hard-tabs", ->
+    beforeEach ->
+      tabLength = 2
+      editSession = fixturesProject.buildEditSessionForPath('sample-with-tabs.coffee', { tabLength })
+      buffer = editSession.buffer
+      tokenizedBuffer = editSession.displayBuffer.tokenizedBuffer
+      editSession.setVisible(true)
+
+    afterEach ->
+      editSession.destroy()
+
+    describe "when the buffer is fully tokenized", ->
+      beforeEach ->
+        fullyTokenize(tokenizedBuffer)
+
+      it "renders each tab as its own atomic token with a value of size tabLength", ->
+        tabAsSpaces = _.multiplyString(' ', editSession.getTabLength())
         screenLine0 = tokenizedBuffer.lineForScreenRow(0)
         expect(screenLine0.text).toBe "# Econ 101#{tabAsSpaces}"
         { tokens } = screenLine0
@@ -178,16 +314,3 @@ describe "TokenizedBuffer", ->
 
         expect(tokenizedBuffer.lineForScreenRow(2).text).toBe "#{tabAsSpaces} buy()#{tabAsSpaces}while supply > demand"
 
-  describe ".setTabLength(tabLength)", ->
-    describe "when the file contains soft tabs", ->
-      it "retokenizes leading whitespace based on the new tab length", ->
-        expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].isAtomic).toBeTruthy()
-        expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].value).toBe "  "
-        expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].isAtomic).toBeTruthy()
-        expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].value).toBe "  "
-
-        tokenizedBuffer.setTabLength(4)
-        expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].isAtomic).toBeTruthy()
-        expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].value).toBe "    "
-        expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].isAtomic).toBeFalsy()
-        expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].value).toBe "  current "

spec/spec-helper.coffee

@@ -10,6 +10,7 @@ RootView = require 'root-view'
 Editor = require 'editor'
 TextMateBundle = require 'text-mate-bundle'
 TextMateTheme = require 'text-mate-theme'
+TokenizedBuffer = require 'tokenized-buffer'
 fs = require 'fs'
 require 'window'
 
@@ -29,6 +30,10 @@ beforeEach ->
   spyOn(window, "clearTimeout").andCallFake window.fakeClearTimeout
   spyOn(File.prototype, "detectResurrectionAfterDelay").andCallFake -> @detectResurrection()
 
+  # make tokenization synchronous
+  TokenizedBuffer.prototype.chunkSize = Infinity
+  spyOn(TokenizedBuffer.prototype, "tokenizeInBackground").andCallFake -> @tokenizeNextChunk()
+
 afterEach ->
   delete window.rootView if window.rootView
   $('#jasmine-content').empty()

src/app/cursor.coffee

@@ -107,7 +107,6 @@ class Cursor
   moveToFirstCharacterOfLine: ->
     position = @getBufferPosition()
     range = @getCurrentLineBufferRange()
-    console.log range.inspect()
     newPosition = null
     @editSession.scanInRange /^\s*/, range, (match, matchRange) =>
       newPosition = matchRange.end

src/app/display-buffer.coffee

@@ -27,6 +27,8 @@ class DisplayBuffer
     @buildLineMap()
     @tokenizedBuffer.on 'change', (e) => @handleTokenizedBufferChange(e)
 
+  setVisible: (visible) -> @tokenizedBuffer.setVisible(visible)
+
   buildLineMap: ->
     @lineMap = new LineMap
     @lineMap.insertAtScreenRow 0, @buildLinesForBufferRows(0, @buffer.getLastRow())

src/app/edit-session.coffee

@@ -91,6 +91,8 @@ class EditSession
       @scrollLeft == other.getScrollLeft() and
       @getCursorScreenPosition().isEqual(other.getCursorScreenPosition())
 
+  setVisible: (visible) -> @displayBuffer.setVisible(visible)
+
   setScrollTop: (@scrollTop) ->
   getScrollTop: -> @scrollTop
 

src/app/editor.coffee

@@ -425,6 +425,7 @@ class Editor extends View
       @activeEditSession.off()
 
     @activeEditSession = @editSessions[index]
+    @activeEditSession.setVisible(true)
 
     @activeEditSession.on "buffer-contents-change-on-disk", =>
       @showBufferConflictAlert(@activeEditSession)

src/app/tokenized-buffer.coffee

@@ -14,63 +14,126 @@ class TokenizedBuffer
   buffer: null
   aceAdaptor: null
   screenLines: null
+  chunkSize: 50
+  invalidRows: null
+  visible: false
 
   constructor: (@buffer, { @languageMode, @tabLength }) ->
     @tabLength ?= 2
     @id = @constructor.idCounter++
-    @screenLines = @buildScreenLinesForRows(0, @buffer.getLastRow())
+    @screenLines = @buildPlaceholderScreenLinesForRows(0, @buffer.getLastRow())
+    @invalidRows = []
+    @invalidateRow(0)
     @buffer.on "change.tokenized-buffer#{@id}", (e) => @handleBufferChange(e)
 
+  setVisible: (@visible) ->
+    @tokenizeInBackground() if @visible
+
+  getTabLength: ->
+    @tabLength
+
+  setTabLength: (@tabLength) ->
+    lastRow = @buffer.getLastRow()
+    @screenLines = @buildPlaceholderScreenLinesForRows(0, lastRow)
+    @invalidateRow(0)
+    @trigger "change", { start: 0, end: lastRow, delta: 0 }
+
+  tokenizeInBackground: ->
+    return if not @visible or @pendingChunk
+    @pendingChunk = true
+    _.defer =>
+      @pendingChunk = false
+      @tokenizeNextChunk()
+
+  tokenizeNextChunk: ->
+    rowsRemaining = @chunkSize
+
+    while @firstInvalidRow()? and rowsRemaining > 0
+      invalidRow = @invalidRows.shift()
+      lastRow = @getLastRow()
+      continue if invalidRow > lastRow
+
+      row = invalidRow
+      loop
+        previousStack = @stackForRow(row)
+        @screenLines[row] = @buildTokenizedScreenLineForRow(row, @stackForRow(row - 1))
+        if --rowsRemaining == 0
+          filledRegion = false
+          break
+        if row == lastRow or _.isEqual(@stackForRow(row), previousStack)
+          filledRegion = true
+          break
+        row++
+
+      @validateRow(row)
+      @invalidateRow(row + 1) unless filledRegion
+      @trigger "change", { start: invalidRow, end: row, delta: 0 }
+
+    @tokenizeInBackground() if @firstInvalidRow()?
+
+  firstInvalidRow: ->
+    @invalidRows[0]
+
+  validateRow: (row) ->
+    @invalidRows.shift() while @invalidRows[0] <= row
+
+  invalidateRow: (row) ->
+    @invalidRows.push(row)
+    @invalidRows.sort (a, b) -> a - b
+    @tokenizeInBackground()
+
+  updateInvalidRows: (start, end, delta) ->
+    @invalidRows = @invalidRows.map (row) ->
+      if row < start
+        row
+      else if start <= row <= end
+        end + delta + 1
+      else if row > end
+        row + delta
+
   handleBufferChange: (e) ->
     {oldRange, newRange} = e
     start = oldRange.start.row
     end = oldRange.end.row
     delta = newRange.end.row - oldRange.end.row
 
+    @updateInvalidRows(start, end, delta)
+
     previousStack = @stackForRow(end) # used in spill detection below
 
     stack = @stackForRow(start - 1)
-    @screenLines[start..end] = @buildScreenLinesForRows(start, end + delta, stack)
+    if stack? or start == 0
+      @screenLines[start..end] = @buildTokenizedScreenLinesForRows(start, end + delta, stack)
+    else
+      @screenLines[start..end] = @buildPlaceholderScreenLinesForRows(start, end + delta, stack)
 
-    # spill detection
-    # compare scanner state of last re-highlighted line with its previous state.
-    # if it differs, re-tokenize the next line with the new state and repeat for
-    # each line until the line's new state matches the previous state. this covers
-    # cases like inserting a /* needing to comment out lines below until we see a */
-    for row in [(end + delta)...@buffer.getLastRow()]
-      break if _.isEqual(@stackForRow(row), previousStack)
-      nextRow = row + 1
-      previousStack = @stackForRow(nextRow)
-      @screenLines[nextRow] = @buildScreenLineForRow(nextRow, @stackForRow(row))
-
-    # if highlighting spilled beyond the bounds of the textual change, update the
-    # end of the affected range to reflect the larger area of highlighting
-    end = Math.max(end, nextRow - delta) if nextRow
+    unless _.isEqual(@stackForRow(end + delta), previousStack)
+      @invalidateRow(end + delta + 1)
 
     @trigger "change", { start, end, delta, bufferChange: e }
 
-  getTabLength: ->
-    @tabLength
+  buildPlaceholderScreenLinesForRows: (startRow, endRow) ->
+    @buildPlaceholderScreenLineForRow(row) for row in [startRow..endRow]
 
-  setTabLength: (@tabLength) ->
-    lastRow = @buffer.getLastRow()
-    @screenLines = @buildScreenLinesForRows(0, lastRow)
-    @trigger "change", { start: 0, end: lastRow, delta: 0 }
+  buildPlaceholderScreenLineForRow: (row) ->
+    line = @buffer.lineForRow(row)
+    tokens = [new Token(value: line, scopes: [@languageMode.grammar.scopeName])]
+    new ScreenLine({tokens, @tabLength})
 
-  buildScreenLinesForRows: (startRow, endRow, startingStack) ->
+  buildTokenizedScreenLinesForRows: (startRow, endRow, startingStack) ->
     ruleStack = startingStack
     for row in [startRow..endRow]
-      screenLine = @buildScreenLineForRow(row, ruleStack)
+      screenLine = @buildTokenizedScreenLineForRow(row, ruleStack)
       ruleStack = screenLine.ruleStack
       screenLine
 
-  buildScreenLineForRow: (row, ruleStack) ->
+  buildTokenizedScreenLineForRow: (row, ruleStack) ->
     line = @buffer.lineForRow(row)
     { tokens, ruleStack } = @languageMode.tokenizeLine(line, ruleStack)
     new ScreenLine({tokens, ruleStack, @tabLength})
 
   lineForScreenRow: (row) ->
-    @screenLines[row]
+    @linesForScreenRows(row, row)[0]
 
   linesForScreenRows: (startRow, endRow) ->
     @screenLines[startRow..endRow]
@@ -146,6 +209,9 @@ class TokenizedBuffer
             stop()
     position
 
+  getLastRow: ->
+    @buffer.getLastRow()
+
   logLines: (start=0, end=@buffer.getLastRow()) ->
     for row in [start..end]
       line = @lineForScreenRow(row).text