diff --git a/spec/app/display-buffer-spec.coffee b/spec/app/display-buffer-spec.coffee index 1443b2d53..fa827ef74 100644 --- a/spec/app/display-buffer-spec.coffee +++ b/spec/app/display-buffer-spec.coffee @@ -565,4 +565,3 @@ describe "DisplayBuffer", -> describe ".maxLineLength()", -> it "returns the length of the longest screen line", -> expect(displayBuffer.maxLineLength()).toBe 65 - diff --git a/spec/app/tokenized-buffer-spec.coffee b/spec/app/tokenized-buffer-spec.coffee index cb8ea03e7..a874e43f0 100644 --- a/spec/app/tokenized-buffer-spec.coffee +++ b/spec/app/tokenized-buffer-spec.coffee @@ -5,166 +5,302 @@ Range = require 'range' _ = require 'underscore' describe "TokenizedBuffer", -> - [editSession, tokenizedBuffer, buffer] = [] + [editSession, tokenizedBuffer, buffer, changeHandler] = [] beforeEach -> - editSession = fixturesProject.buildEditSessionForPath('sample.js', autoIndent: false) - buffer = editSession.buffer - tokenizedBuffer = editSession.displayBuffer.tokenizedBuffer + # enable async tokenization + TokenizedBuffer.prototype.chunkSize = 5 + jasmine.unspy(TokenizedBuffer.prototype, 'tokenizeInBackground') - afterEach -> - editSession.destroy() + fullyTokenize = (tokenizedBuffer) -> + advanceClock() while tokenizedBuffer.firstInvalidRow()? + changeHandler.reset() - describe ".findOpeningBracket(closingBufferPosition)", -> - it "returns the position of the matching bracket, skipping any nested brackets", -> - expect(tokenizedBuffer.findOpeningBracket([9, 2])).toEqual [1, 29] + describe "when the buffer contains soft-tabs", -> + beforeEach -> + editSession = fixturesProject.buildEditSessionForPath('sample.js', autoIndent: false) + buffer = editSession.buffer + tokenizedBuffer = editSession.displayBuffer.tokenizedBuffer + editSession.setVisible(true) + changeHandler = jasmine.createSpy('changeHandler') + tokenizedBuffer.on "change", changeHandler - describe ".findClosingBracket(startBufferPosition)", -> - it "returns the position of the matching bracket, skipping any nested brackets", -> - expect(tokenizedBuffer.findClosingBracket([1, 29])).toEqual [9, 2] + afterEach -> + editSession.destroy() - describe "tokenization", -> - it "tokenizes all the lines in the buffer on construction", -> - expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.modifier.js']) - expect(tokenizedBuffer.lineForScreenRow(11).tokens[1]).toEqual(value: 'return', scopes: ['source.js', 'keyword.control.js']) + describe "on construction", -> + it "initially creates un-tokenized screen lines, then tokenizes lines chunk at a time in the background", -> + line0 = tokenizedBuffer.lineForScreenRow(0) + expect(line0.tokens.length).toBe 1 + expect(line0.tokens[0]).toEqual(value: line0.text, scopes: ['source.js']) - describe "when the buffer changes", -> - changeHandler = null + line11 = tokenizedBuffer.lineForScreenRow(11) + expect(line11.tokens.length).toBe 2 + expect(line11.tokens[0]).toEqual(value: " ", scopes: ['source.js'], isAtomic: true) + expect(line11.tokens[1]).toEqual(value: "return sort(Array.apply(this, arguments));", scopes: ['source.js']) + # background tokenization has not begun + expect(tokenizedBuffer.lineForScreenRow(0).ruleStack).toBeUndefined() + + # tokenize chunk 1 + advanceClock() + expect(tokenizedBuffer.lineForScreenRow(0).ruleStack?).toBeTruthy() + expect(tokenizedBuffer.lineForScreenRow(4).ruleStack?).toBeTruthy() + expect(tokenizedBuffer.lineForScreenRow(5).ruleStack?).toBeFalsy() + expect(changeHandler).toHaveBeenCalledWith(start: 0, end: 4, delta: 0) + changeHandler.reset() + + # tokenize chunk 2 + advanceClock() + expect(tokenizedBuffer.lineForScreenRow(5).ruleStack?).toBeTruthy() + expect(tokenizedBuffer.lineForScreenRow(9).ruleStack?).toBeTruthy() + expect(tokenizedBuffer.lineForScreenRow(10).ruleStack?).toBeFalsy() + expect(changeHandler).toHaveBeenCalledWith(start: 5, end: 9, delta: 0) + changeHandler.reset() + + # tokenize last chunk + advanceClock() + expect(tokenizedBuffer.lineForScreenRow(10).ruleStack?).toBeTruthy() + expect(tokenizedBuffer.lineForScreenRow(12).ruleStack?).toBeTruthy() + expect(changeHandler).toHaveBeenCalledWith(start: 10, end: 12, delta: 0) + + describe "when the buffer is partially tokenized", -> beforeEach -> - changeHandler = jasmine.createSpy('changeHandler') - tokenizedBuffer.on "change", changeHandler + # tokenize chunk 1 only + advanceClock() + changeHandler.reset() - describe "when lines are updated, but none are added or removed", -> - it "updates tokens for each of the changed lines", -> - range = new Range([0, 0], [2, 0]) - buffer.change(range, "foo()\n7\n") + describe "when there is a buffer change inside the tokenized region", -> + describe "when lines are added", -> + it "pushes the invalid rows down", -> + expect(tokenizedBuffer.firstInvalidRow()).toBe 5 + buffer.insert([1, 0], '\n\n') + changeHandler.reset() - expect(tokenizedBuffer.lineForScreenRow(0).tokens[1]).toEqual(value: '(', scopes: ['source.js', 'meta.brace.round.js']) - expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: '7', scopes: ['source.js', 'constant.numeric.js']) - # line 2 is unchanged - expect(tokenizedBuffer.lineForScreenRow(2).tokens[2]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js']) + expect(tokenizedBuffer.firstInvalidRow()).toBe 7 + advanceClock() + expect(changeHandler).toHaveBeenCalledWith(start: 7, end: 11, delta: 0) - expect(changeHandler).toHaveBeenCalled() - [event] = changeHandler.argsForCall[0] - delete event.bufferChange - expect(event).toEqual(start: 0, end: 2, delta: 0) + describe "when lines are removed", -> + it "pulls the invalid rows up", -> + expect(tokenizedBuffer.firstInvalidRow()).toBe 5 + buffer.delete([[1, 0], [3, 0]]) + changeHandler.reset() - it "updates tokens for lines beyond the changed lines if needed", -> - buffer.insert([5, 30], '/* */') + expect(tokenizedBuffer.firstInvalidRow()).toBe 3 + advanceClock() + expect(changeHandler).toHaveBeenCalledWith(start: 3, end: 7, delta: 0) + + describe "when the change invalidates all the lines before the current invalid region", -> + it "retokenizes the invalidated lines and continues into the valid region", -> + expect(tokenizedBuffer.firstInvalidRow()).toBe 5 + buffer.insert([2, 0], '/*') + changeHandler.reset() + expect(tokenizedBuffer.firstInvalidRow()).toBe 3 + + advanceClock() + expect(changeHandler).toHaveBeenCalledWith(start: 3, end: 7, delta: 0) + expect(tokenizedBuffer.firstInvalidRow()).toBe 8 + + describe "when there is a buffer change surrounding an invalid row", -> + it "pushes the invalid row to the end of the change", -> + buffer.change([[4, 0], [6, 0]], "\n\n\n") changeHandler.reset() - buffer.insert([2, 0], '/*') - expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.firstInvalidRow()).toBe 8 + advanceClock() - expect(changeHandler).toHaveBeenCalled() - [event] = changeHandler.argsForCall[0] - delete event.bufferChange - expect(event).toEqual(start: 2, end: 5, delta: 0) + describe "when there is a buffer change inside an invalid region", -> + it "does not attempt to tokenize the lines in the change, and preserves the existing invalid row", -> + expect(tokenizedBuffer.firstInvalidRow()).toBe 5 + buffer.change([[6, 0], [7, 0]], "\n\n\n") - it "resumes highlighting with the state of the previous line", -> - buffer.insert([0, 0], '/*') - buffer.insert([5, 0], '*/') + expect(tokenizedBuffer.lineForScreenRow(6).ruleStack?).toBeFalsy() + expect(tokenizedBuffer.lineForScreenRow(7).ruleStack?).toBeFalsy() - buffer.insert([1, 0], 'var ') - expect(tokenizedBuffer.lineForScreenRow(1).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - - describe "when lines are both updated and removed", -> - it "updates tokens to reflect the removed lines", -> - range = new Range([1, 0], [3, 0]) - buffer.change(range, "foo()") - - # previous line 0 remains - expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.modifier.js']) - - # previous line 3 should be combined with input to form line 1 - expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js']) - expect(tokenizedBuffer.lineForScreenRow(1).tokens[6]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js']) - - # lines below deleted regions should be shifted upward - expect(tokenizedBuffer.lineForScreenRow(2).tokens[2]).toEqual(value: 'while', scopes: ['source.js', 'keyword.control.js']) - expect(tokenizedBuffer.lineForScreenRow(3).tokens[4]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js']) - expect(tokenizedBuffer.lineForScreenRow(4).tokens[4]).toEqual(value: '<', scopes: ['source.js', 'keyword.operator.js']) - - expect(changeHandler).toHaveBeenCalled() - [event] = changeHandler.argsForCall[0] - delete event.bufferChange - expect(event).toEqual(start: 1, end: 3, delta: -2) - - it "updates tokens for lines beyond the changed lines if needed", -> - buffer.insert([5, 30], '/* */') changeHandler.reset() + expect(tokenizedBuffer.firstInvalidRow()).toBe 5 - buffer.change(new Range([2, 0], [3, 0]), '/*') - expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] - expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - - expect(changeHandler).toHaveBeenCalled() - [event] = changeHandler.argsForCall[0] - delete event.bufferChange - expect(event).toEqual(start: 2, end: 5, delta: -1) - - describe "when lines are both updated and inserted", -> - it "updates tokens to reflect the inserted lines", -> - range = new Range([1, 0], [2, 0]) - buffer.change(range, "foo()\nbar()\nbaz()\nquux()") - - # previous line 0 remains - expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual( value: 'var', scopes: ['source.js', 'storage.modifier.js']) - - # 3 new lines inserted - expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js']) - expect(tokenizedBuffer.lineForScreenRow(2).tokens[0]).toEqual(value: 'bar', scopes: ['source.js']) - expect(tokenizedBuffer.lineForScreenRow(3).tokens[0]).toEqual(value: 'baz', scopes: ['source.js']) - - # previous line 2 is joined with quux() on line 4 - expect(tokenizedBuffer.lineForScreenRow(4).tokens[0]).toEqual(value: 'quux', scopes: ['source.js']) - expect(tokenizedBuffer.lineForScreenRow(4).tokens[4]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js']) - - # previous line 3 is pushed down to become line 5 - expect(tokenizedBuffer.lineForScreenRow(5).tokens[4]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js']) - - expect(changeHandler).toHaveBeenCalled() - [event] = changeHandler.argsForCall[0] - delete event.bufferChange - expect(event).toEqual(start: 1, end: 2, delta: 2) - - it "updates tokens for lines beyond the changed lines if needed", -> - buffer.insert([5, 30], '/* */') - changeHandler.reset() - - buffer.insert([2, 0], '/*\nabcde\nabcder') - expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] - expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - expect(tokenizedBuffer.lineForScreenRow(6).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - expect(tokenizedBuffer.lineForScreenRow(7).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - expect(tokenizedBuffer.lineForScreenRow(8).tokens[0].scopes).not.toBe ['source.js', 'comment.block.js'] - - expect(changeHandler).toHaveBeenCalled() - [event] = changeHandler.argsForCall[0] - delete event.bufferChange - expect(event).toEqual(start: 2, end: 5, delta: 2) - - - describe "when the buffer contains tab characters", -> - editSession2 = null - + describe "when the buffer is fully tokenized", -> beforeEach -> - tabLength = 2 - editSession2 = fixturesProject.buildEditSessionForPath('sample-with-tabs.coffee', { tabLength }) - buffer = editSession2.buffer - tokenizedBuffer = editSession2.displayBuffer.tokenizedBuffer + fullyTokenize(tokenizedBuffer) - afterEach -> - editSession2.destroy() + describe "when there is a buffer change that is smaller than the chunk size", -> + describe "when lines are updated, but none are added or removed", -> + it "updates tokens to reflect the change", -> + buffer.change([[0, 0], [2, 0]], "foo()\n7\n") - it "always renders each tab as its own atomic token with a value of size tabLength", -> - tabAsSpaces = _.multiplyString(' ', editSession2.getTabLength()) + expect(tokenizedBuffer.lineForScreenRow(0).tokens[1]).toEqual(value: '(', scopes: ['source.js', 'meta.brace.round.js']) + expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: '7', scopes: ['source.js', 'constant.numeric.js']) + # line 2 is unchanged + expect(tokenizedBuffer.lineForScreenRow(2).tokens[2]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js']) + + expect(changeHandler).toHaveBeenCalled() + [event] = changeHandler.argsForCall[0] + delete event.bufferChange + expect(event).toEqual(start: 0, end: 2, delta: 0) + + describe "when the change invalidates the tokenization of subsequent lines", -> + it "schedules the invalidated lines to be tokenized in the background", -> + buffer.insert([5, 30], '/* */') + changeHandler.reset() + buffer.insert([2, 0], '/*') + expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js'] + expect(changeHandler).toHaveBeenCalled() + [event] = changeHandler.argsForCall[0] + delete event.bufferChange + expect(event).toEqual(start: 2, end: 2, delta: 0) + changeHandler.reset() + + advanceClock() + expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(changeHandler).toHaveBeenCalled() + [event] = changeHandler.argsForCall[0] + delete event.bufferChange + expect(event).toEqual(start: 3, end: 5, delta: 0) + + it "resumes highlighting with the state of the previous line", -> + buffer.insert([0, 0], '/*') + buffer.insert([5, 0], '*/') + + buffer.insert([1, 0], 'var ') + expect(tokenizedBuffer.lineForScreenRow(1).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + + describe "when lines are both updated and removed", -> + it "updates tokens to reflect the change", -> + buffer.change([[1, 0], [3, 0]], "foo()") + + # previous line 0 remains + expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.modifier.js']) + + # previous line 3 should be combined with input to form line 1 + expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js']) + expect(tokenizedBuffer.lineForScreenRow(1).tokens[6]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js']) + + # lines below deleted regions should be shifted upward + expect(tokenizedBuffer.lineForScreenRow(2).tokens[2]).toEqual(value: 'while', scopes: ['source.js', 'keyword.control.js']) + expect(tokenizedBuffer.lineForScreenRow(3).tokens[4]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js']) + expect(tokenizedBuffer.lineForScreenRow(4).tokens[4]).toEqual(value: '<', scopes: ['source.js', 'keyword.operator.js']) + + expect(changeHandler).toHaveBeenCalled() + [event] = changeHandler.argsForCall[0] + delete event.bufferChange + expect(event).toEqual(start: 1, end: 3, delta: -2) + + describe "when the change invalidates the tokenization of subsequent lines", -> + it "schedules the invalidated lines to be tokenized in the background", -> + buffer.insert([5, 30], '/* */') + changeHandler.reset() + + buffer.change([[2, 0], [3, 0]], '/*') + expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] + expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js'] + expect(changeHandler).toHaveBeenCalled() + [event] = changeHandler.argsForCall[0] + delete event.bufferChange + expect(event).toEqual(start: 2, end: 3, delta: -1) + changeHandler.reset() + + advanceClock() + expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(changeHandler).toHaveBeenCalled() + [event] = changeHandler.argsForCall[0] + delete event.bufferChange + expect(event).toEqual(start: 3, end: 4, delta: 0) + + describe "when lines are both updated and inserted", -> + it "updates tokens to reflect the change", -> + buffer.change([[1, 0], [2, 0]], "foo()\nbar()\nbaz()\nquux()") + + # previous line 0 remains + expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual( value: 'var', scopes: ['source.js', 'storage.modifier.js']) + + # 3 new lines inserted + expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js']) + expect(tokenizedBuffer.lineForScreenRow(2).tokens[0]).toEqual(value: 'bar', scopes: ['source.js']) + expect(tokenizedBuffer.lineForScreenRow(3).tokens[0]).toEqual(value: 'baz', scopes: ['source.js']) + + # previous line 2 is joined with quux() on line 4 + expect(tokenizedBuffer.lineForScreenRow(4).tokens[0]).toEqual(value: 'quux', scopes: ['source.js']) + expect(tokenizedBuffer.lineForScreenRow(4).tokens[4]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js']) + + # previous line 3 is pushed down to become line 5 + expect(tokenizedBuffer.lineForScreenRow(5).tokens[4]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js']) + + expect(changeHandler).toHaveBeenCalled() + [event] = changeHandler.argsForCall[0] + delete event.bufferChange + expect(event).toEqual(start: 1, end: 2, delta: 2) + + describe "when the change invalidates the tokenization of subsequent lines", -> + it "schedules the invalidated lines to be tokenized in the background", -> + buffer.insert([5, 30], '/* */') + changeHandler.reset() + + buffer.insert([2, 0], '/*\nabcde\nabcder') + expect(changeHandler).toHaveBeenCalled() + [event] = changeHandler.argsForCall[0] + delete event.bufferChange + expect(event).toEqual(start: 2, end: 2, delta: 2) + expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] + expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js'] + changeHandler.reset() + + advanceClock() # tokenize invalidated lines in background + expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.lineForScreenRow(6).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.lineForScreenRow(7).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.lineForScreenRow(8).tokens[0].scopes).not.toBe ['source.js', 'comment.block.js'] + + expect(changeHandler).toHaveBeenCalled() + [event] = changeHandler.argsForCall[0] + delete event.bufferChange + expect(event).toEqual(start: 5, end: 7, delta: 0) + + describe ".findOpeningBracket(closingBufferPosition)", -> + it "returns the position of the matching bracket, skipping any nested brackets", -> + expect(tokenizedBuffer.findOpeningBracket([9, 2])).toEqual [1, 29] + + describe ".findClosingBracket(startBufferPosition)", -> + it "returns the position of the matching bracket, skipping any nested brackets", -> + expect(tokenizedBuffer.findClosingBracket([1, 29])).toEqual [9, 2] + + it "tokenizes leading whitespace based on the new tab length", -> + expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].isAtomic).toBeTruthy() + expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].value).toBe " " + expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].isAtomic).toBeTruthy() + expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].value).toBe " " + + tokenizedBuffer.setTabLength(4) + fullyTokenize(tokenizedBuffer) + + expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].isAtomic).toBeTruthy() + expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].value).toBe " " + expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].isAtomic).toBeFalsy() + expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].value).toBe " current " + + describe "when the buffer contains hard-tabs", -> + beforeEach -> + tabLength = 2 + editSession = fixturesProject.buildEditSessionForPath('sample-with-tabs.coffee', { tabLength }) + buffer = editSession.buffer + tokenizedBuffer = editSession.displayBuffer.tokenizedBuffer + editSession.setVisible(true) + + afterEach -> + editSession.destroy() + + describe "when the buffer is fully tokenized", -> + beforeEach -> + fullyTokenize(tokenizedBuffer) + + it "renders each tab as its own atomic token with a value of size tabLength", -> + tabAsSpaces = _.multiplyString(' ', editSession.getTabLength()) screenLine0 = tokenizedBuffer.lineForScreenRow(0) expect(screenLine0.text).toBe "# Econ 101#{tabAsSpaces}" { tokens } = screenLine0 @@ -178,16 +314,3 @@ describe "TokenizedBuffer", -> expect(tokenizedBuffer.lineForScreenRow(2).text).toBe "#{tabAsSpaces} buy()#{tabAsSpaces}while supply > demand" - describe ".setTabLength(tabLength)", -> - describe "when the file contains soft tabs", -> - it "retokenizes leading whitespace based on the new tab length", -> - expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].isAtomic).toBeTruthy() - expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].value).toBe " " - expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].isAtomic).toBeTruthy() - expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].value).toBe " " - - tokenizedBuffer.setTabLength(4) - expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].isAtomic).toBeTruthy() - expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].value).toBe " " - expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].isAtomic).toBeFalsy() - expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].value).toBe " current " diff --git a/spec/spec-helper.coffee b/spec/spec-helper.coffee index 885a63aa7..4dfe9713b 100644 --- a/spec/spec-helper.coffee +++ b/spec/spec-helper.coffee @@ -10,6 +10,7 @@ RootView = require 'root-view' Editor = require 'editor' TextMateBundle = require 'text-mate-bundle' TextMateTheme = require 'text-mate-theme' +TokenizedBuffer = require 'tokenized-buffer' fs = require 'fs' require 'window' @@ -29,6 +30,10 @@ beforeEach -> spyOn(window, "clearTimeout").andCallFake window.fakeClearTimeout spyOn(File.prototype, "detectResurrectionAfterDelay").andCallFake -> @detectResurrection() + # make tokenization synchronous + TokenizedBuffer.prototype.chunkSize = Infinity + spyOn(TokenizedBuffer.prototype, "tokenizeInBackground").andCallFake -> @tokenizeNextChunk() + afterEach -> delete window.rootView if window.rootView $('#jasmine-content').empty() diff --git a/src/app/cursor.coffee b/src/app/cursor.coffee index 2e81fa0e2..fa1debab2 100644 --- a/src/app/cursor.coffee +++ b/src/app/cursor.coffee @@ -107,7 +107,6 @@ class Cursor moveToFirstCharacterOfLine: -> position = @getBufferPosition() range = @getCurrentLineBufferRange() - console.log range.inspect() newPosition = null @editSession.scanInRange /^\s*/, range, (match, matchRange) => newPosition = matchRange.end diff --git a/src/app/display-buffer.coffee b/src/app/display-buffer.coffee index 568e59ebe..adf4cfd8a 100644 --- a/src/app/display-buffer.coffee +++ b/src/app/display-buffer.coffee @@ -27,6 +27,8 @@ class DisplayBuffer @buildLineMap() @tokenizedBuffer.on 'change', (e) => @handleTokenizedBufferChange(e) + setVisible: (visible) -> @tokenizedBuffer.setVisible(visible) + buildLineMap: -> @lineMap = new LineMap @lineMap.insertAtScreenRow 0, @buildLinesForBufferRows(0, @buffer.getLastRow()) diff --git a/src/app/edit-session.coffee b/src/app/edit-session.coffee index 6e88924e1..e4b5a42b7 100644 --- a/src/app/edit-session.coffee +++ b/src/app/edit-session.coffee @@ -91,6 +91,8 @@ class EditSession @scrollLeft == other.getScrollLeft() and @getCursorScreenPosition().isEqual(other.getCursorScreenPosition()) + setVisible: (visible) -> @displayBuffer.setVisible(visible) + setScrollTop: (@scrollTop) -> getScrollTop: -> @scrollTop diff --git a/src/app/editor.coffee b/src/app/editor.coffee index 9d0c08fbe..73285250d 100644 --- a/src/app/editor.coffee +++ b/src/app/editor.coffee @@ -425,6 +425,7 @@ class Editor extends View @activeEditSession.off() @activeEditSession = @editSessions[index] + @activeEditSession.setVisible(true) @activeEditSession.on "buffer-contents-change-on-disk", => @showBufferConflictAlert(@activeEditSession) diff --git a/src/app/tokenized-buffer.coffee b/src/app/tokenized-buffer.coffee index 2003c57c7..d1c0cb742 100644 --- a/src/app/tokenized-buffer.coffee +++ b/src/app/tokenized-buffer.coffee @@ -14,63 +14,126 @@ class TokenizedBuffer buffer: null aceAdaptor: null screenLines: null + chunkSize: 50 + invalidRows: null + visible: false constructor: (@buffer, { @languageMode, @tabLength }) -> @tabLength ?= 2 @id = @constructor.idCounter++ - @screenLines = @buildScreenLinesForRows(0, @buffer.getLastRow()) + @screenLines = @buildPlaceholderScreenLinesForRows(0, @buffer.getLastRow()) + @invalidRows = [] + @invalidateRow(0) @buffer.on "change.tokenized-buffer#{@id}", (e) => @handleBufferChange(e) + setVisible: (@visible) -> + @tokenizeInBackground() if @visible + + getTabLength: -> + @tabLength + + setTabLength: (@tabLength) -> + lastRow = @buffer.getLastRow() + @screenLines = @buildPlaceholderScreenLinesForRows(0, lastRow) + @invalidateRow(0) + @trigger "change", { start: 0, end: lastRow, delta: 0 } + + tokenizeInBackground: -> + return if not @visible or @pendingChunk + @pendingChunk = true + _.defer => + @pendingChunk = false + @tokenizeNextChunk() + + tokenizeNextChunk: -> + rowsRemaining = @chunkSize + + while @firstInvalidRow()? and rowsRemaining > 0 + invalidRow = @invalidRows.shift() + lastRow = @getLastRow() + continue if invalidRow > lastRow + + row = invalidRow + loop + previousStack = @stackForRow(row) + @screenLines[row] = @buildTokenizedScreenLineForRow(row, @stackForRow(row - 1)) + if --rowsRemaining == 0 + filledRegion = false + break + if row == lastRow or _.isEqual(@stackForRow(row), previousStack) + filledRegion = true + break + row++ + + @validateRow(row) + @invalidateRow(row + 1) unless filledRegion + @trigger "change", { start: invalidRow, end: row, delta: 0 } + + @tokenizeInBackground() if @firstInvalidRow()? + + firstInvalidRow: -> + @invalidRows[0] + + validateRow: (row) -> + @invalidRows.shift() while @invalidRows[0] <= row + + invalidateRow: (row) -> + @invalidRows.push(row) + @invalidRows.sort (a, b) -> a - b + @tokenizeInBackground() + + updateInvalidRows: (start, end, delta) -> + @invalidRows = @invalidRows.map (row) -> + if row < start + row + else if start <= row <= end + end + delta + 1 + else if row > end + row + delta + handleBufferChange: (e) -> {oldRange, newRange} = e start = oldRange.start.row end = oldRange.end.row delta = newRange.end.row - oldRange.end.row + @updateInvalidRows(start, end, delta) + previousStack = @stackForRow(end) # used in spill detection below stack = @stackForRow(start - 1) - @screenLines[start..end] = @buildScreenLinesForRows(start, end + delta, stack) + if stack? or start == 0 + @screenLines[start..end] = @buildTokenizedScreenLinesForRows(start, end + delta, stack) + else + @screenLines[start..end] = @buildPlaceholderScreenLinesForRows(start, end + delta, stack) - # spill detection - # compare scanner state of last re-highlighted line with its previous state. - # if it differs, re-tokenize the next line with the new state and repeat for - # each line until the line's new state matches the previous state. this covers - # cases like inserting a /* needing to comment out lines below until we see a */ - for row in [(end + delta)...@buffer.getLastRow()] - break if _.isEqual(@stackForRow(row), previousStack) - nextRow = row + 1 - previousStack = @stackForRow(nextRow) - @screenLines[nextRow] = @buildScreenLineForRow(nextRow, @stackForRow(row)) - - # if highlighting spilled beyond the bounds of the textual change, update the - # end of the affected range to reflect the larger area of highlighting - end = Math.max(end, nextRow - delta) if nextRow + unless _.isEqual(@stackForRow(end + delta), previousStack) + @invalidateRow(end + delta + 1) @trigger "change", { start, end, delta, bufferChange: e } - getTabLength: -> - @tabLength + buildPlaceholderScreenLinesForRows: (startRow, endRow) -> + @buildPlaceholderScreenLineForRow(row) for row in [startRow..endRow] - setTabLength: (@tabLength) -> - lastRow = @buffer.getLastRow() - @screenLines = @buildScreenLinesForRows(0, lastRow) - @trigger "change", { start: 0, end: lastRow, delta: 0 } + buildPlaceholderScreenLineForRow: (row) -> + line = @buffer.lineForRow(row) + tokens = [new Token(value: line, scopes: [@languageMode.grammar.scopeName])] + new ScreenLine({tokens, @tabLength}) - buildScreenLinesForRows: (startRow, endRow, startingStack) -> + buildTokenizedScreenLinesForRows: (startRow, endRow, startingStack) -> ruleStack = startingStack for row in [startRow..endRow] - screenLine = @buildScreenLineForRow(row, ruleStack) + screenLine = @buildTokenizedScreenLineForRow(row, ruleStack) ruleStack = screenLine.ruleStack screenLine - buildScreenLineForRow: (row, ruleStack) -> + buildTokenizedScreenLineForRow: (row, ruleStack) -> line = @buffer.lineForRow(row) { tokens, ruleStack } = @languageMode.tokenizeLine(line, ruleStack) new ScreenLine({tokens, ruleStack, @tabLength}) lineForScreenRow: (row) -> - @screenLines[row] + @linesForScreenRows(row, row)[0] linesForScreenRows: (startRow, endRow) -> @screenLines[startRow..endRow] @@ -146,6 +209,9 @@ class TokenizedBuffer stop() position + getLastRow: -> + @buffer.getLastRow() + logLines: (start=0, end=@buffer.getLastRow()) -> for row in [start..end] line = @lineForScreenRow(row).text