From e52dbb682e3b9664e43510810f6c732c4e6c5627 Mon Sep 17 00:00:00 2001 From: Corey Johnson & Nathan Sobo Date: Mon, 19 Nov 2012 16:50:49 -0700 Subject: [PATCH 01/16] WIP: Perform tokenization in chunks when edit session is shown --- spec/app/tokenized-buffer-spec.coffee | 47 +++++++++++++++++++++++---- src/app/display-buffer.coffee | 3 ++ src/app/edit-session.coffee | 3 ++ src/app/editor.coffee | 1 + src/app/tokenized-buffer.coffee | 45 ++++++++++++++++++++----- 5 files changed, 84 insertions(+), 15 deletions(-) diff --git a/spec/app/tokenized-buffer-spec.coffee b/spec/app/tokenized-buffer-spec.coffee index cb8ea03e7..f33c138cf 100644 --- a/spec/app/tokenized-buffer-spec.coffee +++ b/spec/app/tokenized-buffer-spec.coffee @@ -5,12 +5,14 @@ Range = require 'range' _ = require 'underscore' describe "TokenizedBuffer", -> - [editSession, tokenizedBuffer, buffer] = [] + [editSession, tokenizedBuffer, buffer, changeHandler] = [] beforeEach -> editSession = fixturesProject.buildEditSessionForPath('sample.js', autoIndent: false) buffer = editSession.buffer tokenizedBuffer = editSession.displayBuffer.tokenizedBuffer + changeHandler = jasmine.createSpy('changeHandler') + tokenizedBuffer.on "change", changeHandler afterEach -> editSession.destroy() @@ -24,17 +26,48 @@ describe "TokenizedBuffer", -> expect(tokenizedBuffer.findClosingBracket([1, 29])).toEqual [9, 2] describe "tokenization", -> + it "only creates untokenized screen lines on construction", -> + line0 = tokenizedBuffer.lineForScreenRow(0) + expect(line0.tokens.length).toBe 1 + expect(line0.tokens[0]).toEqual(value: line0.text, scopes: ['source.js']) + + line11 = tokenizedBuffer.lineForScreenRow(11) + expect(line11.tokens.length).toBe 2 + expect(line11.tokens[0]).toEqual(value: " ", scopes: ['source.js'], isAtomic: true) + expect(line11.tokens[1]).toEqual(value: "return sort(Array.apply(this, arguments));", scopes: ['source.js']) + + describe "when #tokenizeInBackground() is called", -> + it "tokenizes screen lines one chunk at a time asynchronously after calling #activate()", -> + tokenizedBuffer.chunkSize = 5 + tokenizedBuffer.tokenizeInBackground() + + line0 = tokenizedBuffer.lineForScreenRow(0) # kicks off tokenization + expect(line0.ruleStack).toBeUndefined() + + advanceClock() # trigger deferred code + expect(tokenizedBuffer.lineForScreenRow(0).ruleStack?).toBeTruthy() + expect(tokenizedBuffer.lineForScreenRow(4).ruleStack?).toBeTruthy() + expect(tokenizedBuffer.lineForScreenRow(5).ruleStack?).toBeFalsy() + expect(changeHandler).toHaveBeenCalledWith(start: 0, end: 4, delta: 0) + changeHandler.reset() + + advanceClock() # trigger deferred code again + expect(tokenizedBuffer.lineForScreenRow(5).ruleStack?).toBeTruthy() + expect(tokenizedBuffer.lineForScreenRow(9).ruleStack?).toBeTruthy() + expect(tokenizedBuffer.lineForScreenRow(10).ruleStack?).toBeFalsy() + expect(changeHandler).toHaveBeenCalledWith(start: 5, end: 9, delta: 0) + changeHandler.reset() + + advanceClock() # trigger deferred code again + expect(tokenizedBuffer.lineForScreenRow(10).ruleStack?).toBeTruthy() + expect(tokenizedBuffer.lineForScreenRow(12).ruleStack?).toBeTruthy() + expect(changeHandler).toHaveBeenCalledWith(start: 10, end: 12, delta: 0) + it "tokenizes all the lines in the buffer on construction", -> expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.modifier.js']) expect(tokenizedBuffer.lineForScreenRow(11).tokens[1]).toEqual(value: 'return', scopes: ['source.js', 'keyword.control.js']) describe "when the buffer changes", -> - changeHandler = null - - beforeEach -> - changeHandler = jasmine.createSpy('changeHandler') - tokenizedBuffer.on "change", changeHandler - describe "when lines are updated, but none are added or removed", -> it "updates tokens for each of the changed lines", -> range = new Range([0, 0], [2, 0]) diff --git a/src/app/display-buffer.coffee b/src/app/display-buffer.coffee index 568e59ebe..28c68f03d 100644 --- a/src/app/display-buffer.coffee +++ b/src/app/display-buffer.coffee @@ -39,6 +39,9 @@ class DisplayBuffer bufferDelta = 0 @trigger 'change', { start, end, screenDelta, bufferDelta } + tokenizeInBackground: -> + @tokenizedBuffer.tokenizeInBackground() + lineForRow: (row) -> @lineMap.lineForScreenRow(row) diff --git a/src/app/edit-session.coffee b/src/app/edit-session.coffee index 6e88924e1..002d2d75a 100644 --- a/src/app/edit-session.coffee +++ b/src/app/edit-session.coffee @@ -84,6 +84,9 @@ class EditSession copy: -> EditSession.deserialize(@serialize(), @project) + activate: -> + @displayBuffer.tokenizeInBackground() + isEqual: (other) -> return false unless other instanceof EditSession @buffer == other.buffer and diff --git a/src/app/editor.coffee b/src/app/editor.coffee index adb2592fa..2bae92ee8 100644 --- a/src/app/editor.coffee +++ b/src/app/editor.coffee @@ -422,6 +422,7 @@ class Editor extends View @activeEditSession.off() @activeEditSession = @editSessions[index] + @activeEditSession.activate() @activeEditSession.on "buffer-contents-change-on-disk", => @showBufferConflictAlert(@activeEditSession) diff --git a/src/app/tokenized-buffer.coffee b/src/app/tokenized-buffer.coffee index 2003c57c7..c528e32c2 100644 --- a/src/app/tokenized-buffer.coffee +++ b/src/app/tokenized-buffer.coffee @@ -14,11 +14,13 @@ class TokenizedBuffer buffer: null aceAdaptor: null screenLines: null + untokenizedRow: 0 + chunkSize: 50 constructor: (@buffer, { @languageMode, @tabLength }) -> @tabLength ?= 2 @id = @constructor.idCounter++ - @screenLines = @buildScreenLinesForRows(0, @buffer.getLastRow()) + @screenLines = @buildPlaceholderScreenLinesForRows(0, @buffer.getLastRow()) @buffer.on "change.tokenized-buffer#{@id}", (e) => @handleBufferChange(e) handleBufferChange: (e) -> @@ -30,7 +32,7 @@ class TokenizedBuffer previousStack = @stackForRow(end) # used in spill detection below stack = @stackForRow(start - 1) - @screenLines[start..end] = @buildScreenLinesForRows(start, end + delta, stack) + @screenLines[start..end] = @buildPlaceholderScreenLinesForRows(start, end + delta, stack) # spill detection # compare scanner state of last re-highlighted line with its previous state. @@ -41,7 +43,7 @@ class TokenizedBuffer break if _.isEqual(@stackForRow(row), previousStack) nextRow = row + 1 previousStack = @stackForRow(nextRow) - @screenLines[nextRow] = @buildScreenLineForRow(nextRow, @stackForRow(row)) + @screenLines[nextRow] = @buildTokenizedScreenLineForRow(nextRow, @stackForRow(row)) # if highlighting spilled beyond the bounds of the textual change, update the # end of the affected range to reflect the larger area of highlighting @@ -54,25 +56,52 @@ class TokenizedBuffer setTabLength: (@tabLength) -> lastRow = @buffer.getLastRow() - @screenLines = @buildScreenLinesForRows(0, lastRow) + @screenLines = @buildPlaceholderScreenLinesForRows(0, lastRow) @trigger "change", { start: 0, end: lastRow, delta: 0 } - buildScreenLinesForRows: (startRow, endRow, startingStack) -> + tokenizeInBackground: -> + return if @tokenizingInBackground + @tokenizingInBackground = true + _.defer => @tokenizeNextChunk() + + tokenizeNextChunk: -> + lastRow = @buffer.getLastRow() + stack = @stackForRow(@untokenizedRow - 1) + start = @untokenizedRow + end = Math.min(start + @chunkSize - 1, lastRow) + @screenLines[start..end] = @buildTokenizedScreenLinesForRows(start, end, stack) + @trigger "change", { start, end, delta: 0} + + @untokenizedRow = end + 1 + + if @untokenizedRow <= lastRow + _.defer => @tokenizeNextChunk() + + buildPlaceholderScreenLinesForRows: (startRow, endRow) -> + @buildPlaceholderScreenLineForRow(row) for row in [startRow..endRow] + + buildPlaceholderScreenLineForRow: (row) -> + line = @buffer.lineForRow(row) + tokens = [new Token(value: line, scopes: [@languageMode.grammar.scopeName])] + new ScreenLine({tokens, @tabLength}) + + buildTokenizedScreenLinesForRows: (startRow, endRow, startingStack) -> ruleStack = startingStack for row in [startRow..endRow] - screenLine = @buildScreenLineForRow(row, ruleStack) + screenLine = @buildTokenizedScreenLineForRow(row, ruleStack) ruleStack = screenLine.ruleStack screenLine - buildScreenLineForRow: (row, ruleStack) -> + buildTokenizedScreenLineForRow: (row, ruleStack) -> line = @buffer.lineForRow(row) { tokens, ruleStack } = @languageMode.tokenizeLine(line, ruleStack) new ScreenLine({tokens, ruleStack, @tabLength}) lineForScreenRow: (row) -> - @screenLines[row] + @linesForScreenRows(row, row)[0] linesForScreenRows: (startRow, endRow) -> + @tokenizeInBackground() @screenLines[startRow..endRow] stackForRow: (row) -> From 53c7ccb29748b1643ca2f997f3aff9de8715f5b3 Mon Sep 17 00:00:00 2001 From: Corey Johnson & Nathan Sobo Date: Mon, 19 Nov 2012 17:11:20 -0700 Subject: [PATCH 02/16] WIP: Fleshing out specs for async tokenization --- spec/app/tokenized-buffer-spec.coffee | 195 ++++++++++++++------------ 1 file changed, 104 insertions(+), 91 deletions(-) diff --git a/spec/app/tokenized-buffer-spec.coffee b/spec/app/tokenized-buffer-spec.coffee index f33c138cf..fd6b8b512 100644 --- a/spec/app/tokenized-buffer-spec.coffee +++ b/spec/app/tokenized-buffer-spec.coffee @@ -67,121 +67,134 @@ describe "TokenizedBuffer", -> expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.modifier.js']) expect(tokenizedBuffer.lineForScreenRow(11).tokens[1]).toEqual(value: 'return', scopes: ['source.js', 'keyword.control.js']) - describe "when the buffer changes", -> - describe "when lines are updated, but none are added or removed", -> - it "updates tokens for each of the changed lines", -> - range = new Range([0, 0], [2, 0]) - buffer.change(range, "foo()\n7\n") + describe "when there is a buffer change starting in the tokenized region", -> + describe "when the change causes fewer than @chunkSize lines to be retokenized", -> + describe "when lines are updated, but none are added or removed", -> + it "updates tokens for each of the changed lines", -> + range = new Range([0, 0], [2, 0]) + buffer.change(range, "foo()\n7\n") - expect(tokenizedBuffer.lineForScreenRow(0).tokens[1]).toEqual(value: '(', scopes: ['source.js', 'meta.brace.round.js']) - expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: '7', scopes: ['source.js', 'constant.numeric.js']) - # line 2 is unchanged - expect(tokenizedBuffer.lineForScreenRow(2).tokens[2]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js']) + expect(tokenizedBuffer.lineForScreenRow(0).tokens[1]).toEqual(value: '(', scopes: ['source.js', 'meta.brace.round.js']) + expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: '7', scopes: ['source.js', 'constant.numeric.js']) + # line 2 is unchanged + expect(tokenizedBuffer.lineForScreenRow(2).tokens[2]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js']) - expect(changeHandler).toHaveBeenCalled() - [event] = changeHandler.argsForCall[0] - delete event.bufferChange - expect(event).toEqual(start: 0, end: 2, delta: 0) + expect(changeHandler).toHaveBeenCalled() + [event] = changeHandler.argsForCall[0] + delete event.bufferChange + expect(event).toEqual(start: 0, end: 2, delta: 0) - it "updates tokens for lines beyond the changed lines if needed", -> - buffer.insert([5, 30], '/* */') - changeHandler.reset() + it "updates tokens for lines beyond the changed lines if needed", -> + buffer.insert([5, 30], '/* */') + changeHandler.reset() - buffer.insert([2, 0], '/*') - expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + buffer.insert([2, 0], '/*') + expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - expect(changeHandler).toHaveBeenCalled() - [event] = changeHandler.argsForCall[0] - delete event.bufferChange - expect(event).toEqual(start: 2, end: 5, delta: 0) + expect(changeHandler).toHaveBeenCalled() + [event] = changeHandler.argsForCall[0] + delete event.bufferChange + expect(event).toEqual(start: 2, end: 5, delta: 0) - it "resumes highlighting with the state of the previous line", -> - buffer.insert([0, 0], '/*') - buffer.insert([5, 0], '*/') + it "resumes highlighting with the state of the previous line", -> + buffer.insert([0, 0], '/*') + buffer.insert([5, 0], '*/') - buffer.insert([1, 0], 'var ') - expect(tokenizedBuffer.lineForScreenRow(1).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + buffer.insert([1, 0], 'var ') + expect(tokenizedBuffer.lineForScreenRow(1).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - describe "when lines are both updated and removed", -> - it "updates tokens to reflect the removed lines", -> - range = new Range([1, 0], [3, 0]) - buffer.change(range, "foo()") + describe "when lines are both updated and removed", -> + it "updates tokens to reflect the removed lines", -> + range = new Range([1, 0], [3, 0]) + buffer.change(range, "foo()") - # previous line 0 remains - expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.modifier.js']) + # previous line 0 remains + expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.modifier.js']) - # previous line 3 should be combined with input to form line 1 - expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js']) - expect(tokenizedBuffer.lineForScreenRow(1).tokens[6]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js']) + # previous line 3 should be combined with input to form line 1 + expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js']) + expect(tokenizedBuffer.lineForScreenRow(1).tokens[6]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js']) - # lines below deleted regions should be shifted upward - expect(tokenizedBuffer.lineForScreenRow(2).tokens[2]).toEqual(value: 'while', scopes: ['source.js', 'keyword.control.js']) - expect(tokenizedBuffer.lineForScreenRow(3).tokens[4]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js']) - expect(tokenizedBuffer.lineForScreenRow(4).tokens[4]).toEqual(value: '<', scopes: ['source.js', 'keyword.operator.js']) + # lines below deleted regions should be shifted upward + expect(tokenizedBuffer.lineForScreenRow(2).tokens[2]).toEqual(value: 'while', scopes: ['source.js', 'keyword.control.js']) + expect(tokenizedBuffer.lineForScreenRow(3).tokens[4]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js']) + expect(tokenizedBuffer.lineForScreenRow(4).tokens[4]).toEqual(value: '<', scopes: ['source.js', 'keyword.operator.js']) - expect(changeHandler).toHaveBeenCalled() - [event] = changeHandler.argsForCall[0] - delete event.bufferChange - expect(event).toEqual(start: 1, end: 3, delta: -2) + expect(changeHandler).toHaveBeenCalled() + [event] = changeHandler.argsForCall[0] + delete event.bufferChange + expect(event).toEqual(start: 1, end: 3, delta: -2) - it "updates tokens for lines beyond the changed lines if needed", -> - buffer.insert([5, 30], '/* */') - changeHandler.reset() + it "updates tokens for lines beyond the changed lines if needed", -> + buffer.insert([5, 30], '/* */') + changeHandler.reset() - buffer.change(new Range([2, 0], [3, 0]), '/*') - expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] - expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + buffer.change(new Range([2, 0], [3, 0]), '/*') + expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] + expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - expect(changeHandler).toHaveBeenCalled() - [event] = changeHandler.argsForCall[0] - delete event.bufferChange - expect(event).toEqual(start: 2, end: 5, delta: -1) + expect(changeHandler).toHaveBeenCalled() + [event] = changeHandler.argsForCall[0] + delete event.bufferChange + expect(event).toEqual(start: 2, end: 5, delta: -1) - describe "when lines are both updated and inserted", -> - it "updates tokens to reflect the inserted lines", -> - range = new Range([1, 0], [2, 0]) - buffer.change(range, "foo()\nbar()\nbaz()\nquux()") + describe "when lines are both updated and inserted", -> + it "updates tokens to reflect the inserted lines", -> + range = new Range([1, 0], [2, 0]) + buffer.change(range, "foo()\nbar()\nbaz()\nquux()") - # previous line 0 remains - expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual( value: 'var', scopes: ['source.js', 'storage.modifier.js']) + # previous line 0 remains + expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual( value: 'var', scopes: ['source.js', 'storage.modifier.js']) - # 3 new lines inserted - expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js']) - expect(tokenizedBuffer.lineForScreenRow(2).tokens[0]).toEqual(value: 'bar', scopes: ['source.js']) - expect(tokenizedBuffer.lineForScreenRow(3).tokens[0]).toEqual(value: 'baz', scopes: ['source.js']) + # 3 new lines inserted + expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js']) + expect(tokenizedBuffer.lineForScreenRow(2).tokens[0]).toEqual(value: 'bar', scopes: ['source.js']) + expect(tokenizedBuffer.lineForScreenRow(3).tokens[0]).toEqual(value: 'baz', scopes: ['source.js']) - # previous line 2 is joined with quux() on line 4 - expect(tokenizedBuffer.lineForScreenRow(4).tokens[0]).toEqual(value: 'quux', scopes: ['source.js']) - expect(tokenizedBuffer.lineForScreenRow(4).tokens[4]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js']) + # previous line 2 is joined with quux() on line 4 + expect(tokenizedBuffer.lineForScreenRow(4).tokens[0]).toEqual(value: 'quux', scopes: ['source.js']) + expect(tokenizedBuffer.lineForScreenRow(4).tokens[4]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js']) - # previous line 3 is pushed down to become line 5 - expect(tokenizedBuffer.lineForScreenRow(5).tokens[4]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js']) + # previous line 3 is pushed down to become line 5 + expect(tokenizedBuffer.lineForScreenRow(5).tokens[4]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js']) - expect(changeHandler).toHaveBeenCalled() - [event] = changeHandler.argsForCall[0] - delete event.bufferChange - expect(event).toEqual(start: 1, end: 2, delta: 2) + expect(changeHandler).toHaveBeenCalled() + [event] = changeHandler.argsForCall[0] + delete event.bufferChange + expect(event).toEqual(start: 1, end: 2, delta: 2) - it "updates tokens for lines beyond the changed lines if needed", -> - buffer.insert([5, 30], '/* */') - changeHandler.reset() + it "updates tokens for lines beyond the changed lines if needed", -> + buffer.insert([5, 30], '/* */') + changeHandler.reset() - buffer.insert([2, 0], '/*\nabcde\nabcder') - expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] - expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - expect(tokenizedBuffer.lineForScreenRow(6).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - expect(tokenizedBuffer.lineForScreenRow(7).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - expect(tokenizedBuffer.lineForScreenRow(8).tokens[0].scopes).not.toBe ['source.js', 'comment.block.js'] + buffer.insert([2, 0], '/*\nabcde\nabcder') + expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] + expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.lineForScreenRow(6).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.lineForScreenRow(7).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.lineForScreenRow(8).tokens[0].scopes).not.toBe ['source.js', 'comment.block.js'] - expect(changeHandler).toHaveBeenCalled() - [event] = changeHandler.argsForCall[0] - delete event.bufferChange - expect(event).toEqual(start: 2, end: 5, delta: 2) + expect(changeHandler).toHaveBeenCalled() + [event] = changeHandler.argsForCall[0] + delete event.bufferChange + expect(event).toEqual(start: 2, end: 5, delta: 2) + + describe "when the change causes more than @chunkSize lines to be retokenized", -> + describe "when the buffer change itself is larger than the chunk size", -> + + describe "when the buffer change is smaller than the chunk size, but the scope of rehighlighting is larger", -> + + describe "when the end of rehighlighted region is outside the tokenized region", -> + it "extends the boundary of the tokenized region and resumes tokenizing after it", -> + + + describe "when there is a buffer change in the untokenized region", -> + it "updates the untokenized screen lines to match the buffer but does not tokenize the lines", -> describe "when the buffer contains tab characters", -> From b011c0ab8815e964be48b62b490ab14108f364ba Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Tue, 20 Nov 2012 13:06:01 -0700 Subject: [PATCH 03/16] Kill `console.log` --- src/app/cursor.coffee | 1 - 1 file changed, 1 deletion(-) diff --git a/src/app/cursor.coffee b/src/app/cursor.coffee index 2e81fa0e2..fa1debab2 100644 --- a/src/app/cursor.coffee +++ b/src/app/cursor.coffee @@ -107,7 +107,6 @@ class Cursor moveToFirstCharacterOfLine: -> position = @getBufferPosition() range = @getCurrentLineBufferRange() - console.log range.inspect() newPosition = null @editSession.scanInRange /^\s*/, range, (match, matchRange) => newPosition = matchRange.end From 5aba8596a9261b902e8ec49776d68690db23509e Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Wed, 21 Nov 2012 08:22:13 -0700 Subject: [PATCH 04/16] Make tokenization synchronous in all specs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Disabled some specs that need to make it asynchronous again… will deal with those soon. --- spec/app/tokenized-buffer-spec.coffee | 4 ++-- spec/spec-helper.coffee | 5 +++++ src/app/display-buffer.coffee | 3 --- src/app/edit-session.coffee | 3 --- src/app/editor.coffee | 1 - src/app/tokenized-buffer.coffee | 19 +++++++++++-------- 6 files changed, 18 insertions(+), 17 deletions(-) diff --git a/spec/app/tokenized-buffer-spec.coffee b/spec/app/tokenized-buffer-spec.coffee index fd6b8b512..e34c8c414 100644 --- a/spec/app/tokenized-buffer-spec.coffee +++ b/spec/app/tokenized-buffer-spec.coffee @@ -26,7 +26,7 @@ describe "TokenizedBuffer", -> expect(tokenizedBuffer.findClosingBracket([1, 29])).toEqual [9, 2] describe "tokenization", -> - it "only creates untokenized screen lines on construction", -> + xit "only creates untokenized screen lines on construction", -> line0 = tokenizedBuffer.lineForScreenRow(0) expect(line0.tokens.length).toBe 1 expect(line0.tokens[0]).toEqual(value: line0.text, scopes: ['source.js']) @@ -36,7 +36,7 @@ describe "TokenizedBuffer", -> expect(line11.tokens[0]).toEqual(value: " ", scopes: ['source.js'], isAtomic: true) expect(line11.tokens[1]).toEqual(value: "return sort(Array.apply(this, arguments));", scopes: ['source.js']) - describe "when #tokenizeInBackground() is called", -> + xdescribe "when #tokenizeInBackground() is called", -> it "tokenizes screen lines one chunk at a time asynchronously after calling #activate()", -> tokenizedBuffer.chunkSize = 5 tokenizedBuffer.tokenizeInBackground() diff --git a/spec/spec-helper.coffee b/spec/spec-helper.coffee index 885a63aa7..4dfe9713b 100644 --- a/spec/spec-helper.coffee +++ b/spec/spec-helper.coffee @@ -10,6 +10,7 @@ RootView = require 'root-view' Editor = require 'editor' TextMateBundle = require 'text-mate-bundle' TextMateTheme = require 'text-mate-theme' +TokenizedBuffer = require 'tokenized-buffer' fs = require 'fs' require 'window' @@ -29,6 +30,10 @@ beforeEach -> spyOn(window, "clearTimeout").andCallFake window.fakeClearTimeout spyOn(File.prototype, "detectResurrectionAfterDelay").andCallFake -> @detectResurrection() + # make tokenization synchronous + TokenizedBuffer.prototype.chunkSize = Infinity + spyOn(TokenizedBuffer.prototype, "tokenizeInBackground").andCallFake -> @tokenizeNextChunk() + afterEach -> delete window.rootView if window.rootView $('#jasmine-content').empty() diff --git a/src/app/display-buffer.coffee b/src/app/display-buffer.coffee index 28c68f03d..568e59ebe 100644 --- a/src/app/display-buffer.coffee +++ b/src/app/display-buffer.coffee @@ -39,9 +39,6 @@ class DisplayBuffer bufferDelta = 0 @trigger 'change', { start, end, screenDelta, bufferDelta } - tokenizeInBackground: -> - @tokenizedBuffer.tokenizeInBackground() - lineForRow: (row) -> @lineMap.lineForScreenRow(row) diff --git a/src/app/edit-session.coffee b/src/app/edit-session.coffee index 002d2d75a..6e88924e1 100644 --- a/src/app/edit-session.coffee +++ b/src/app/edit-session.coffee @@ -84,9 +84,6 @@ class EditSession copy: -> EditSession.deserialize(@serialize(), @project) - activate: -> - @displayBuffer.tokenizeInBackground() - isEqual: (other) -> return false unless other instanceof EditSession @buffer == other.buffer and diff --git a/src/app/editor.coffee b/src/app/editor.coffee index 2bae92ee8..adb2592fa 100644 --- a/src/app/editor.coffee +++ b/src/app/editor.coffee @@ -422,7 +422,6 @@ class Editor extends View @activeEditSession.off() @activeEditSession = @editSessions[index] - @activeEditSession.activate() @activeEditSession.on "buffer-contents-change-on-disk", => @showBufferConflictAlert(@activeEditSession) diff --git a/src/app/tokenized-buffer.coffee b/src/app/tokenized-buffer.coffee index c528e32c2..de23c1c20 100644 --- a/src/app/tokenized-buffer.coffee +++ b/src/app/tokenized-buffer.coffee @@ -21,6 +21,7 @@ class TokenizedBuffer @tabLength ?= 2 @id = @constructor.idCounter++ @screenLines = @buildPlaceholderScreenLinesForRows(0, @buffer.getLastRow()) + @tokenizeInBackground() @buffer.on "change.tokenized-buffer#{@id}", (e) => @handleBufferChange(e) handleBufferChange: (e) -> @@ -32,7 +33,7 @@ class TokenizedBuffer previousStack = @stackForRow(end) # used in spill detection below stack = @stackForRow(start - 1) - @screenLines[start..end] = @buildPlaceholderScreenLinesForRows(start, end + delta, stack) + @screenLines[start..end] = @buildTokenizedScreenLinesForRows(start, end + delta, stack) # spill detection # compare scanner state of last re-highlighted line with its previous state. @@ -56,26 +57,29 @@ class TokenizedBuffer setTabLength: (@tabLength) -> lastRow = @buffer.getLastRow() + @untokenizedRow = 0 @screenLines = @buildPlaceholderScreenLinesForRows(0, lastRow) + @tokenizeInBackground() @trigger "change", { start: 0, end: lastRow, delta: 0 } tokenizeInBackground: -> - return if @tokenizingInBackground - @tokenizingInBackground = true - _.defer => @tokenizeNextChunk() + return if @pendingChunk or @untokenizedRow > @buffer.getLastRow() + @pendingChunk = true + _.defer => + @pendingChunk = false + @tokenizeNextChunk() tokenizeNextChunk: -> lastRow = @buffer.getLastRow() stack = @stackForRow(@untokenizedRow - 1) start = @untokenizedRow end = Math.min(start + @chunkSize - 1, lastRow) + @screenLines[start..end] = @buildTokenizedScreenLinesForRows(start, end, stack) @trigger "change", { start, end, delta: 0} @untokenizedRow = end + 1 - - if @untokenizedRow <= lastRow - _.defer => @tokenizeNextChunk() + @tokenizeInBackground() if @untokenizedRow <= lastRow buildPlaceholderScreenLinesForRows: (startRow, endRow) -> @buildPlaceholderScreenLineForRow(row) for row in [startRow..endRow] @@ -101,7 +105,6 @@ class TokenizedBuffer @linesForScreenRows(row, row)[0] linesForScreenRows: (startRow, endRow) -> - @tokenizeInBackground() @screenLines[startRow..endRow] stackForRow: (row) -> From e92d9c5c9d81d27af24553b9dd0021396f284c7f Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Wed, 21 Nov 2012 09:09:08 -0700 Subject: [PATCH 05/16] Reorganize TokenizedBuffer spec It's going to have to cover edge cases in the async tokenization, and the previous structure wasn't going to cut it. --- spec/app/tokenized-buffer-spec.coffee | 151 ++++++++++++++------------ src/app/tokenized-buffer.coffee | 5 +- 2 files changed, 83 insertions(+), 73 deletions(-) diff --git a/spec/app/tokenized-buffer-spec.coffee b/spec/app/tokenized-buffer-spec.coffee index e34c8c414..62201c35f 100644 --- a/spec/app/tokenized-buffer-spec.coffee +++ b/spec/app/tokenized-buffer-spec.coffee @@ -4,75 +4,84 @@ Buffer = require 'buffer' Range = require 'range' _ = require 'underscore' -describe "TokenizedBuffer", -> +fdescribe "TokenizedBuffer", -> [editSession, tokenizedBuffer, buffer, changeHandler] = [] beforeEach -> - editSession = fixturesProject.buildEditSessionForPath('sample.js', autoIndent: false) - buffer = editSession.buffer - tokenizedBuffer = editSession.displayBuffer.tokenizedBuffer - changeHandler = jasmine.createSpy('changeHandler') - tokenizedBuffer.on "change", changeHandler + # enable async tokenization + TokenizedBuffer.prototype.chunkSize = 5 + jasmine.unspy(TokenizedBuffer.prototype, 'tokenizeInBackground') - afterEach -> - editSession.destroy() + fullyTokenize = (tokenizedBuffer) -> + advanceClock() while tokenizedBuffer.untokenizedRow <= tokenizedBuffer.getLastRow() + changeHandler.reset() - describe ".findOpeningBracket(closingBufferPosition)", -> - it "returns the position of the matching bracket, skipping any nested brackets", -> - expect(tokenizedBuffer.findOpeningBracket([9, 2])).toEqual [1, 29] + describe "when the buffer contains soft-tabs", -> + beforeEach -> + editSession = fixturesProject.buildEditSessionForPath('sample.js', autoIndent: false) + buffer = editSession.buffer + tokenizedBuffer = editSession.displayBuffer.tokenizedBuffer + changeHandler = jasmine.createSpy('changeHandler') + tokenizedBuffer.on "change", changeHandler - describe ".findClosingBracket(startBufferPosition)", -> - it "returns the position of the matching bracket, skipping any nested brackets", -> - expect(tokenizedBuffer.findClosingBracket([1, 29])).toEqual [9, 2] + afterEach -> + editSession.destroy() - describe "tokenization", -> - xit "only creates untokenized screen lines on construction", -> - line0 = tokenizedBuffer.lineForScreenRow(0) - expect(line0.tokens.length).toBe 1 - expect(line0.tokens[0]).toEqual(value: line0.text, scopes: ['source.js']) + describe "on construction", -> + it "initially creates un-tokenized screen lines, then tokenizes lines chunk at a time in the background", -> + line0 = tokenizedBuffer.lineForScreenRow(0) + expect(line0.tokens.length).toBe 1 + expect(line0.tokens[0]).toEqual(value: line0.text, scopes: ['source.js']) - line11 = tokenizedBuffer.lineForScreenRow(11) - expect(line11.tokens.length).toBe 2 - expect(line11.tokens[0]).toEqual(value: " ", scopes: ['source.js'], isAtomic: true) - expect(line11.tokens[1]).toEqual(value: "return sort(Array.apply(this, arguments));", scopes: ['source.js']) + line11 = tokenizedBuffer.lineForScreenRow(11) + expect(line11.tokens.length).toBe 2 + expect(line11.tokens[0]).toEqual(value: " ", scopes: ['source.js'], isAtomic: true) + expect(line11.tokens[1]).toEqual(value: "return sort(Array.apply(this, arguments));", scopes: ['source.js']) - xdescribe "when #tokenizeInBackground() is called", -> - it "tokenizes screen lines one chunk at a time asynchronously after calling #activate()", -> - tokenizedBuffer.chunkSize = 5 - tokenizedBuffer.tokenizeInBackground() + # background tokenization has not begun + expect(tokenizedBuffer.lineForScreenRow(0).ruleStack).toBeUndefined() - line0 = tokenizedBuffer.lineForScreenRow(0) # kicks off tokenization - expect(line0.ruleStack).toBeUndefined() - - advanceClock() # trigger deferred code + # tokenize chunk 1 + advanceClock() expect(tokenizedBuffer.lineForScreenRow(0).ruleStack?).toBeTruthy() expect(tokenizedBuffer.lineForScreenRow(4).ruleStack?).toBeTruthy() expect(tokenizedBuffer.lineForScreenRow(5).ruleStack?).toBeFalsy() expect(changeHandler).toHaveBeenCalledWith(start: 0, end: 4, delta: 0) changeHandler.reset() - advanceClock() # trigger deferred code again + # tokenize chunk 2 + advanceClock() expect(tokenizedBuffer.lineForScreenRow(5).ruleStack?).toBeTruthy() expect(tokenizedBuffer.lineForScreenRow(9).ruleStack?).toBeTruthy() expect(tokenizedBuffer.lineForScreenRow(10).ruleStack?).toBeFalsy() expect(changeHandler).toHaveBeenCalledWith(start: 5, end: 9, delta: 0) changeHandler.reset() - advanceClock() # trigger deferred code again + # tokenize last chunk + advanceClock() expect(tokenizedBuffer.lineForScreenRow(10).ruleStack?).toBeTruthy() expect(tokenizedBuffer.lineForScreenRow(12).ruleStack?).toBeTruthy() expect(changeHandler).toHaveBeenCalledWith(start: 10, end: 12, delta: 0) - it "tokenizes all the lines in the buffer on construction", -> - expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.modifier.js']) - expect(tokenizedBuffer.lineForScreenRow(11).tokens[1]).toEqual(value: 'return', scopes: ['source.js', 'keyword.control.js']) + describe "when the buffer is partially tokenized", -> + beforeEach -> + # tokenize chunk 1 only + advanceClock() - describe "when there is a buffer change starting in the tokenized region", -> - describe "when the change causes fewer than @chunkSize lines to be retokenized", -> + describe "when there is a buffer change inside a tokenized region", -> + + describe "when there is a buffer change surrounding an invalid row", -> + + describe "when there is a buffer change inside an invalid region", -> + + describe "when the buffer is fully tokenized", -> + beforeEach -> + fullyTokenize(tokenizedBuffer) + + describe "when there is a buffer change that is smaller than the chunk size", -> describe "when lines are updated, but none are added or removed", -> - it "updates tokens for each of the changed lines", -> - range = new Range([0, 0], [2, 0]) - buffer.change(range, "foo()\n7\n") + it "updates tokens to reflect the change", -> + buffer.change([[0, 0], [2, 0]], "foo()\n7\n") expect(tokenizedBuffer.lineForScreenRow(0).tokens[1]).toEqual(value: '(', scopes: ['source.js', 'meta.brace.round.js']) expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: '7', scopes: ['source.js', 'constant.numeric.js']) @@ -184,33 +193,44 @@ describe "TokenizedBuffer", -> delete event.bufferChange expect(event).toEqual(start: 2, end: 5, delta: 2) - describe "when the change causes more than @chunkSize lines to be retokenized", -> - describe "when the buffer change itself is larger than the chunk size", -> + describe ".findOpeningBracket(closingBufferPosition)", -> + it "returns the position of the matching bracket, skipping any nested brackets", -> + expect(tokenizedBuffer.findOpeningBracket([9, 2])).toEqual [1, 29] - describe "when the buffer change is smaller than the chunk size, but the scope of rehighlighting is larger", -> + describe ".findClosingBracket(startBufferPosition)", -> + it "returns the position of the matching bracket, skipping any nested brackets", -> + expect(tokenizedBuffer.findClosingBracket([1, 29])).toEqual [9, 2] - describe "when the end of rehighlighted region is outside the tokenized region", -> - it "extends the boundary of the tokenized region and resumes tokenizing after it", -> + it "tokenizes leading whitespace based on the new tab length", -> + expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].isAtomic).toBeTruthy() + expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].value).toBe " " + expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].isAtomic).toBeTruthy() + expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].value).toBe " " + tokenizedBuffer.setTabLength(4) + fullyTokenize(tokenizedBuffer) - describe "when there is a buffer change in the untokenized region", -> - it "updates the untokenized screen lines to match the buffer but does not tokenize the lines", -> + expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].isAtomic).toBeTruthy() + expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].value).toBe " " + expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].isAtomic).toBeFalsy() + expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].value).toBe " current " + describe "when the buffer contains hard-tabs", -> + beforeEach -> + tabLength = 2 + editSession = fixturesProject.buildEditSessionForPath('sample-with-tabs.coffee', { tabLength }) + buffer = editSession.buffer + tokenizedBuffer = editSession.displayBuffer.tokenizedBuffer - describe "when the buffer contains tab characters", -> - editSession2 = null + afterEach -> + editSession.destroy() + describe "when the buffer is fully tokenized", -> beforeEach -> - tabLength = 2 - editSession2 = fixturesProject.buildEditSessionForPath('sample-with-tabs.coffee', { tabLength }) - buffer = editSession2.buffer - tokenizedBuffer = editSession2.displayBuffer.tokenizedBuffer + fullyTokenize(tokenizedBuffer) - afterEach -> - editSession2.destroy() - - it "always renders each tab as its own atomic token with a value of size tabLength", -> - tabAsSpaces = _.multiplyString(' ', editSession2.getTabLength()) + it "renders each tab as its own atomic token with a value of size tabLength", -> + tabAsSpaces = _.multiplyString(' ', editSession.getTabLength()) screenLine0 = tokenizedBuffer.lineForScreenRow(0) expect(screenLine0.text).toBe "# Econ 101#{tabAsSpaces}" { tokens } = screenLine0 @@ -224,16 +244,3 @@ describe "TokenizedBuffer", -> expect(tokenizedBuffer.lineForScreenRow(2).text).toBe "#{tabAsSpaces} buy()#{tabAsSpaces}while supply > demand" - describe ".setTabLength(tabLength)", -> - describe "when the file contains soft tabs", -> - it "retokenizes leading whitespace based on the new tab length", -> - expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].isAtomic).toBeTruthy() - expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].value).toBe " " - expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].isAtomic).toBeTruthy() - expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].value).toBe " " - - tokenizedBuffer.setTabLength(4) - expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].isAtomic).toBeTruthy() - expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].value).toBe " " - expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].isAtomic).toBeFalsy() - expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].value).toBe " current " diff --git a/src/app/tokenized-buffer.coffee b/src/app/tokenized-buffer.coffee index de23c1c20..4790cba0f 100644 --- a/src/app/tokenized-buffer.coffee +++ b/src/app/tokenized-buffer.coffee @@ -33,6 +33,7 @@ class TokenizedBuffer previousStack = @stackForRow(end) # used in spill detection below stack = @stackForRow(start - 1) + @screenLines[start..end] = @buildTokenizedScreenLinesForRows(start, end + delta, stack) # spill detection @@ -49,7 +50,6 @@ class TokenizedBuffer # if highlighting spilled beyond the bounds of the textual change, update the # end of the affected range to reflect the larger area of highlighting end = Math.max(end, nextRow - delta) if nextRow - @trigger "change", { start, end, delta, bufferChange: e } getTabLength: -> @@ -178,6 +178,9 @@ class TokenizedBuffer stop() position + getLastRow: -> + @buffer.getLastRow() + logLines: (start=0, end=@buffer.getLastRow()) -> for row in [start..end] line = @lineForScreenRow(row).text From 5acd1b6ee309036bfb47a832e3a457d6b2466084 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Wed, 21 Nov 2012 10:02:52 -0700 Subject: [PATCH 06/16] When a change invalidates subsequent lines, re-tokenize asynchronously This can happen when inserting a quote at the top of the file. It switches all the strings to source and vice versa, throughout the file. This can be very laggy, so it's good to do it asynchronously. --- spec/app/tokenized-buffer-spec.coffee | 31 +++++++++------ src/app/tokenized-buffer.coffee | 55 ++++++++++++++++++--------- 2 files changed, 57 insertions(+), 29 deletions(-) diff --git a/spec/app/tokenized-buffer-spec.coffee b/spec/app/tokenized-buffer-spec.coffee index 62201c35f..32dd9c8c1 100644 --- a/spec/app/tokenized-buffer-spec.coffee +++ b/spec/app/tokenized-buffer-spec.coffee @@ -93,19 +93,26 @@ fdescribe "TokenizedBuffer", -> delete event.bufferChange expect(event).toEqual(start: 0, end: 2, delta: 0) - it "updates tokens for lines beyond the changed lines if needed", -> - buffer.insert([5, 30], '/* */') - changeHandler.reset() + describe "when the change invalidates the tokenization of subsequent lines", -> + it "schedules the invalidated lines to be tokenized in the background", -> + buffer.insert([5, 30], '/* */') + changeHandler.reset() + buffer.insert([2, 0], '/*') + expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js'] + expect(changeHandler).toHaveBeenCalled() + [event] = changeHandler.argsForCall[0] + delete event.bufferChange + expect(event).toEqual(start: 2, end: 2, delta: 0) + changeHandler.reset() - buffer.insert([2, 0], '/*') - expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - - expect(changeHandler).toHaveBeenCalled() - [event] = changeHandler.argsForCall[0] - delete event.bufferChange - expect(event).toEqual(start: 2, end: 5, delta: 0) + advanceClock() + expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(changeHandler).toHaveBeenCalled() + [event] = changeHandler.argsForCall[0] + delete event.bufferChange + expect(event).toEqual(start: 3, end: 5, delta: 0) it "resumes highlighting with the state of the previous line", -> buffer.insert([0, 0], '/*') diff --git a/src/app/tokenized-buffer.coffee b/src/app/tokenized-buffer.coffee index 4790cba0f..f2e3a171b 100644 --- a/src/app/tokenized-buffer.coffee +++ b/src/app/tokenized-buffer.coffee @@ -14,13 +14,14 @@ class TokenizedBuffer buffer: null aceAdaptor: null screenLines: null - untokenizedRow: 0 chunkSize: 50 + invalidRows: null constructor: (@buffer, { @languageMode, @tabLength }) -> @tabLength ?= 2 @id = @constructor.idCounter++ @screenLines = @buildPlaceholderScreenLinesForRows(0, @buffer.getLastRow()) + @invalidRows = [0] @tokenizeInBackground() @buffer.on "change.tokenized-buffer#{@id}", (e) => @handleBufferChange(e) @@ -41,15 +42,22 @@ class TokenizedBuffer # if it differs, re-tokenize the next line with the new state and repeat for # each line until the line's new state matches the previous state. this covers # cases like inserting a /* needing to comment out lines below until we see a */ - for row in [(end + delta)...@buffer.getLastRow()] - break if _.isEqual(@stackForRow(row), previousStack) - nextRow = row + 1 - previousStack = @stackForRow(nextRow) - @screenLines[nextRow] = @buildTokenizedScreenLineForRow(nextRow, @stackForRow(row)) + + + unless _.isEqual(@stackForRow(end + delta), previousStack) + console.log "spill" + @invalidRows.unshift(end + 1) + @tokenizeInBackground() + +# for row in [(end + delta)...@buffer.getLastRow()] +# +# nextRow = row + 1 +# previousStack = @stackForRow(nextRow) +# @screenLines[nextRow] = @buildTokenizedScreenLineForRow(nextRow, @stackForRow(row)) # if highlighting spilled beyond the bounds of the textual change, update the # end of the affected range to reflect the larger area of highlighting - end = Math.max(end, nextRow - delta) if nextRow +# end = Math.max(end, nextRow - delta) if nextRow @trigger "change", { start, end, delta, bufferChange: e } getTabLength: -> @@ -57,29 +65,42 @@ class TokenizedBuffer setTabLength: (@tabLength) -> lastRow = @buffer.getLastRow() - @untokenizedRow = 0 + @invalidRows = [0] @screenLines = @buildPlaceholderScreenLinesForRows(0, lastRow) @tokenizeInBackground() @trigger "change", { start: 0, end: lastRow, delta: 0 } tokenizeInBackground: -> - return if @pendingChunk or @untokenizedRow > @buffer.getLastRow() + return if @pendingChunk @pendingChunk = true _.defer => @pendingChunk = false @tokenizeNextChunk() tokenizeNextChunk: -> - lastRow = @buffer.getLastRow() - stack = @stackForRow(@untokenizedRow - 1) - start = @untokenizedRow - end = Math.min(start + @chunkSize - 1, lastRow) + rowsRemaining = @chunkSize - @screenLines[start..end] = @buildTokenizedScreenLinesForRows(start, end, stack) - @trigger "change", { start, end, delta: 0} + while @invalidRows.length and rowsRemaining > 0 + invalidRow = @invalidRows.shift() + lastRow = @getLastRow() + continue if invalidRow > lastRow - @untokenizedRow = end + 1 - @tokenizeInBackground() if @untokenizedRow <= lastRow + filledRegion = false + row = invalidRow + loop + previousStack = @stackForRow(row) + @screenLines[row] = @buildTokenizedScreenLineForRow(row, @stackForRow(row - 1)) + if row == lastRow or _.isEqual(@stackForRow(row), previousStack) + filledRegion = true + break + if --rowsRemaining == 0 + break + row++ + + @trigger "change", { start: invalidRow, end: row, delta: 0} + @invalidRows.unshift(row + 1) unless filledRegion + + @tokenizeInBackground() buildPlaceholderScreenLinesForRows: (startRow, endRow) -> @buildPlaceholderScreenLineForRow(row) for row in [startRow..endRow] From 0fd921bb40cc8b8f7bbcc1dbb1eaa3fb7b087120 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Wed, 21 Nov 2012 18:38:39 -0700 Subject: [PATCH 07/16] :lipstick: and update specs for async tokenization of invalidated rows --- spec/app/tokenized-buffer-spec.coffee | 46 ++++++++++++++++++--------- src/app/tokenized-buffer.coffee | 40 +++++++++-------------- 2 files changed, 46 insertions(+), 40 deletions(-) diff --git a/spec/app/tokenized-buffer-spec.coffee b/spec/app/tokenized-buffer-spec.coffee index 32dd9c8c1..b7532e9a3 100644 --- a/spec/app/tokenized-buffer-spec.coffee +++ b/spec/app/tokenized-buffer-spec.coffee @@ -13,7 +13,7 @@ fdescribe "TokenizedBuffer", -> jasmine.unspy(TokenizedBuffer.prototype, 'tokenizeInBackground') fullyTokenize = (tokenizedBuffer) -> - advanceClock() while tokenizedBuffer.untokenizedRow <= tokenizedBuffer.getLastRow() + advanceClock() while tokenizedBuffer.firstInvalidRow()? changeHandler.reset() describe "when the buffer contains soft-tabs", -> @@ -76,6 +76,7 @@ fdescribe "TokenizedBuffer", -> describe "when the buffer is fully tokenized", -> beforeEach -> + console.log "FULLY TOKENIZE" fullyTokenize(tokenizedBuffer) describe "when there is a buffer change that is smaller than the chunk size", -> @@ -122,9 +123,8 @@ fdescribe "TokenizedBuffer", -> expect(tokenizedBuffer.lineForScreenRow(1).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] describe "when lines are both updated and removed", -> - it "updates tokens to reflect the removed lines", -> - range = new Range([1, 0], [3, 0]) - buffer.change(range, "foo()") + it "updates tokens to reflect the change", -> + buffer.change([[1, 0], [3, 0]], "foo()") # previous line 0 remains expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.modifier.js']) @@ -143,24 +143,31 @@ fdescribe "TokenizedBuffer", -> delete event.bufferChange expect(event).toEqual(start: 1, end: 3, delta: -2) - it "updates tokens for lines beyond the changed lines if needed", -> + describe "when the change invalidates the tokenization of subsequent lines", -> + it "schedules the invalidated lines to be tokenized in the background", -> buffer.insert([5, 30], '/* */') changeHandler.reset() - buffer.change(new Range([2, 0], [3, 0]), '/*') + buffer.change([[2, 0], [3, 0]], '/*') expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] - expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - + expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js'] expect(changeHandler).toHaveBeenCalled() [event] = changeHandler.argsForCall[0] delete event.bufferChange - expect(event).toEqual(start: 2, end: 5, delta: -1) + expect(event).toEqual(start: 2, end: 3, delta: -1) + changeHandler.reset() + + advanceClock() + expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(changeHandler).toHaveBeenCalled() + [event] = changeHandler.argsForCall[0] + delete event.bufferChange + expect(event).toEqual(start: 3, end: 4, delta: 0) describe "when lines are both updated and inserted", -> - it "updates tokens to reflect the inserted lines", -> - range = new Range([1, 0], [2, 0]) - buffer.change(range, "foo()\nbar()\nbaz()\nquux()") + it "updates tokens to reflect the change", -> + buffer.change([[1, 0], [2, 0]], "foo()\nbar()\nbaz()\nquux()") # previous line 0 remains expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual( value: 'var', scopes: ['source.js', 'storage.modifier.js']) @@ -182,14 +189,23 @@ fdescribe "TokenizedBuffer", -> delete event.bufferChange expect(event).toEqual(start: 1, end: 2, delta: 2) - it "updates tokens for lines beyond the changed lines if needed", -> + describe "when the change invalidates the tokenization of subsequent lines", -> + it "schedules the invalidated lines to be tokenized in the background", -> buffer.insert([5, 30], '/* */') changeHandler.reset() buffer.insert([2, 0], '/*\nabcde\nabcder') + expect(changeHandler).toHaveBeenCalled() + [event] = changeHandler.argsForCall[0] + delete event.bufferChange + expect(event).toEqual(start: 2, end: 2, delta: 2) expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js'] + changeHandler.reset() + + advanceClock() # tokenize invalidated lines in background expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] expect(tokenizedBuffer.lineForScreenRow(6).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] expect(tokenizedBuffer.lineForScreenRow(7).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] @@ -198,7 +214,7 @@ fdescribe "TokenizedBuffer", -> expect(changeHandler).toHaveBeenCalled() [event] = changeHandler.argsForCall[0] delete event.bufferChange - expect(event).toEqual(start: 2, end: 5, delta: 2) + expect(event).toEqual(start: 5, end: 7, delta: 0) describe ".findOpeningBracket(closingBufferPosition)", -> it "returns the position of the matching bracket, skipping any nested brackets", -> diff --git a/src/app/tokenized-buffer.coffee b/src/app/tokenized-buffer.coffee index f2e3a171b..53647ba2a 100644 --- a/src/app/tokenized-buffer.coffee +++ b/src/app/tokenized-buffer.coffee @@ -21,8 +21,8 @@ class TokenizedBuffer @tabLength ?= 2 @id = @constructor.idCounter++ @screenLines = @buildPlaceholderScreenLinesForRows(0, @buffer.getLastRow()) - @invalidRows = [0] - @tokenizeInBackground() + @invalidRows = [] + @invalidateRow(0) @buffer.on "change.tokenized-buffer#{@id}", (e) => @handleBufferChange(e) handleBufferChange: (e) -> @@ -37,27 +37,10 @@ class TokenizedBuffer @screenLines[start..end] = @buildTokenizedScreenLinesForRows(start, end + delta, stack) - # spill detection - # compare scanner state of last re-highlighted line with its previous state. - # if it differs, re-tokenize the next line with the new state and repeat for - # each line until the line's new state matches the previous state. this covers - # cases like inserting a /* needing to comment out lines below until we see a */ - - unless _.isEqual(@stackForRow(end + delta), previousStack) console.log "spill" - @invalidRows.unshift(end + 1) - @tokenizeInBackground() + @invalidateRow(end + delta + 1) -# for row in [(end + delta)...@buffer.getLastRow()] -# -# nextRow = row + 1 -# previousStack = @stackForRow(nextRow) -# @screenLines[nextRow] = @buildTokenizedScreenLineForRow(nextRow, @stackForRow(row)) - - # if highlighting spilled beyond the bounds of the textual change, update the - # end of the affected range to reflect the larger area of highlighting -# end = Math.max(end, nextRow - delta) if nextRow @trigger "change", { start, end, delta, bufferChange: e } getTabLength: -> @@ -65,9 +48,8 @@ class TokenizedBuffer setTabLength: (@tabLength) -> lastRow = @buffer.getLastRow() - @invalidRows = [0] @screenLines = @buildPlaceholderScreenLinesForRows(0, lastRow) - @tokenizeInBackground() + @invalidateRow(0) @trigger "change", { start: 0, end: lastRow, delta: 0 } tokenizeInBackground: -> @@ -90,15 +72,15 @@ class TokenizedBuffer loop previousStack = @stackForRow(row) @screenLines[row] = @buildTokenizedScreenLineForRow(row, @stackForRow(row - 1)) + if --rowsRemaining == 0 + break if row == lastRow or _.isEqual(@stackForRow(row), previousStack) filledRegion = true break - if --rowsRemaining == 0 - break row++ @trigger "change", { start: invalidRow, end: row, delta: 0} - @invalidRows.unshift(row + 1) unless filledRegion + @invalidateRow(row + 1) unless filledRegion @tokenizeInBackground() @@ -202,6 +184,14 @@ class TokenizedBuffer getLastRow: -> @buffer.getLastRow() + firstInvalidRow: -> + @invalidRows[0] + + invalidateRow: (row) -> + @invalidRows.push(row) + @invalidRows.sort() + @tokenizeInBackground() + logLines: (start=0, end=@buffer.getLastRow()) -> for row in [start..end] line = @lineForScreenRow(row).text From 893e7428c6749f670ec06d43c22fb852503aeab8 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Wed, 21 Nov 2012 18:44:42 -0700 Subject: [PATCH 08/16] :lipstick: --- spec/app/tokenized-buffer-spec.coffee | 1 - 1 file changed, 1 deletion(-) diff --git a/spec/app/tokenized-buffer-spec.coffee b/spec/app/tokenized-buffer-spec.coffee index b7532e9a3..26227a34b 100644 --- a/spec/app/tokenized-buffer-spec.coffee +++ b/spec/app/tokenized-buffer-spec.coffee @@ -76,7 +76,6 @@ fdescribe "TokenizedBuffer", -> describe "when the buffer is fully tokenized", -> beforeEach -> - console.log "FULLY TOKENIZE" fullyTokenize(tokenizedBuffer) describe "when there is a buffer change that is smaller than the chunk size", -> From 48f9f70dd2621a3ec6a008a79a38960715e74c5a Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Thu, 22 Nov 2012 10:07:24 -0700 Subject: [PATCH 09/16] Handle changes in the valid region of a partially tokenized buffer --- spec/app/tokenized-buffer-spec.coffee | 33 ++++++++++++++++++++++++++- src/app/tokenized-buffer.coffee | 15 +++++++++--- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/spec/app/tokenized-buffer-spec.coffee b/spec/app/tokenized-buffer-spec.coffee index 26227a34b..b6a1dfdf4 100644 --- a/spec/app/tokenized-buffer-spec.coffee +++ b/spec/app/tokenized-buffer-spec.coffee @@ -67,8 +67,39 @@ fdescribe "TokenizedBuffer", -> beforeEach -> # tokenize chunk 1 only advanceClock() + changeHandler.reset() - describe "when there is a buffer change inside a tokenized region", -> + describe "when there is a buffer change inside the tokenized region", -> + describe "when lines are added", -> + it "pushes the invalid rows down", -> + expect(tokenizedBuffer.firstInvalidRow()).toBe 5 + buffer.insert([1, 0], '\n\n') + changeHandler.reset() + + expect(tokenizedBuffer.firstInvalidRow()).toBe 7 + advanceClock() + expect(changeHandler).toHaveBeenCalledWith(start: 7, end: 11, delta: 0) + + describe "when lines are removed", -> + it "pulls the invalid rows up", -> + expect(tokenizedBuffer.firstInvalidRow()).toBe 5 + buffer.delete([[1, 0], [3, 0]]) + changeHandler.reset() + + expect(tokenizedBuffer.firstInvalidRow()).toBe 3 + advanceClock() + expect(changeHandler).toHaveBeenCalledWith(start: 3, end: 7, delta: 0) + + describe "when the change invalidates all the lines before the current invalid region", -> + it "retokenizes the invalidated lines and continues into the valid region", -> + expect(tokenizedBuffer.firstInvalidRow()).toBe 5 + buffer.insert([2, 0], '/*') + changeHandler.reset() + expect(tokenizedBuffer.firstInvalidRow()).toBe 3 + + advanceClock() + expect(changeHandler).toHaveBeenCalledWith(start: 3, end: 7, delta: 0) + expect(tokenizedBuffer.firstInvalidRow()).toBe 8 describe "when there is a buffer change surrounding an invalid row", -> diff --git a/src/app/tokenized-buffer.coffee b/src/app/tokenized-buffer.coffee index 53647ba2a..3160830d7 100644 --- a/src/app/tokenized-buffer.coffee +++ b/src/app/tokenized-buffer.coffee @@ -31,14 +31,13 @@ class TokenizedBuffer end = oldRange.end.row delta = newRange.end.row - oldRange.end.row + @updateInvalidRows(start, end, delta) + previousStack = @stackForRow(end) # used in spill detection below - stack = @stackForRow(start - 1) - @screenLines[start..end] = @buildTokenizedScreenLinesForRows(start, end + delta, stack) unless _.isEqual(@stackForRow(end + delta), previousStack) - console.log "spill" @invalidateRow(end + delta + 1) @trigger "change", { start, end, delta, bufferChange: e } @@ -72,6 +71,7 @@ class TokenizedBuffer loop previousStack = @stackForRow(row) @screenLines[row] = @buildTokenizedScreenLineForRow(row, @stackForRow(row - 1)) + @validateRow(row) if --rowsRemaining == 0 break if row == lastRow or _.isEqual(@stackForRow(row), previousStack) @@ -192,6 +192,15 @@ class TokenizedBuffer @invalidRows.sort() @tokenizeInBackground() + validateRow: (row) -> + @invalidRows.shift() if @invalidRows[0] == row + + updateInvalidRows: (start, end, delta) -> + updatedRows = [] + for row in @invalidRows + updatedRows.push(row + delta) + @invalidRows = updatedRows + logLines: (start=0, end=@buffer.getLastRow()) -> for row in [start..end] line = @lineForScreenRow(row).text From bcab21709be33368613a05223778837e53b1eb7b Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Thu, 22 Nov 2012 11:02:14 -0700 Subject: [PATCH 10/16] Handle changes that straddle or are inside of the invalid regions --- spec/app/tokenized-buffer-spec.coffee | 15 +++++++++++++++ src/app/tokenized-buffer.coffee | 16 +++++++++++++--- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/spec/app/tokenized-buffer-spec.coffee b/spec/app/tokenized-buffer-spec.coffee index b6a1dfdf4..91346de85 100644 --- a/spec/app/tokenized-buffer-spec.coffee +++ b/spec/app/tokenized-buffer-spec.coffee @@ -102,8 +102,23 @@ fdescribe "TokenizedBuffer", -> expect(tokenizedBuffer.firstInvalidRow()).toBe 8 describe "when there is a buffer change surrounding an invalid row", -> + it "pushes the invalid row to the end of the change", -> + buffer.change([[4, 0], [6, 0]], "\n\n\n") + changeHandler.reset() + + expect(tokenizedBuffer.firstInvalidRow()).toBe 8 + advanceClock() describe "when there is a buffer change inside an invalid region", -> + it "does not attempt to tokenize the lines in the change, and preserves the existing invalid row", -> + expect(tokenizedBuffer.firstInvalidRow()).toBe 5 + buffer.change([[6, 0], [7, 0]], "\n\n\n") + + expect(tokenizedBuffer.lineForScreenRow(6).ruleStack?).toBeFalsy() + expect(tokenizedBuffer.lineForScreenRow(7).ruleStack?).toBeFalsy() + + changeHandler.reset() + expect(tokenizedBuffer.firstInvalidRow()).toBe 5 describe "when the buffer is fully tokenized", -> beforeEach -> diff --git a/src/app/tokenized-buffer.coffee b/src/app/tokenized-buffer.coffee index 3160830d7..d585d4bb2 100644 --- a/src/app/tokenized-buffer.coffee +++ b/src/app/tokenized-buffer.coffee @@ -34,8 +34,12 @@ class TokenizedBuffer @updateInvalidRows(start, end, delta) previousStack = @stackForRow(end) # used in spill detection below + stack = @stackForRow(start - 1) - @screenLines[start..end] = @buildTokenizedScreenLinesForRows(start, end + delta, stack) + if stack? or start == 0 + @screenLines[start..end] = @buildTokenizedScreenLinesForRows(start, end + delta, stack) + else + @screenLines[start..end] = @buildPlaceholderScreenLinesForRows(start, end + delta, stack) unless _.isEqual(@stackForRow(end + delta), previousStack) @invalidateRow(end + delta + 1) @@ -189,7 +193,7 @@ class TokenizedBuffer invalidateRow: (row) -> @invalidRows.push(row) - @invalidRows.sort() + @invalidRows.sort (a, b) -> a - b @tokenizeInBackground() validateRow: (row) -> @@ -198,7 +202,13 @@ class TokenizedBuffer updateInvalidRows: (start, end, delta) -> updatedRows = [] for row in @invalidRows - updatedRows.push(row + delta) + if row < start + updatedRows.push(row) + else if start <= row <= end + updatedRows.push(end + delta + 1) + else if row > end + updatedRows.push(row + delta) + @invalidRows = updatedRows logLines: (start=0, end=@buffer.getLastRow()) -> From 640bf6dc8d5d5822648cd89265dd355e2d9f6294 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Fri, 23 Nov 2012 10:37:14 -0700 Subject: [PATCH 11/16] Refactor background tokenization --- src/app/tokenized-buffer.coffee | 49 ++++++++++++++++----------------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/src/app/tokenized-buffer.coffee b/src/app/tokenized-buffer.coffee index d585d4bb2..510223eb0 100644 --- a/src/app/tokenized-buffer.coffee +++ b/src/app/tokenized-buffer.coffee @@ -70,24 +70,44 @@ class TokenizedBuffer lastRow = @getLastRow() continue if invalidRow > lastRow - filledRegion = false row = invalidRow loop previousStack = @stackForRow(row) @screenLines[row] = @buildTokenizedScreenLineForRow(row, @stackForRow(row - 1)) - @validateRow(row) if --rowsRemaining == 0 + filledRegion = false break if row == lastRow or _.isEqual(@stackForRow(row), previousStack) filledRegion = true break row++ - @trigger "change", { start: invalidRow, end: row, delta: 0} + @validateRow(row) @invalidateRow(row + 1) unless filledRegion + @trigger "change", { start: invalidRow, end: row, delta: 0 } @tokenizeInBackground() + firstInvalidRow: -> + @invalidRows[0] + + validateRow: (row) -> + @invalidRows.shift() while @invalidRows[0] <= row + + invalidateRow: (row) -> + @invalidRows.push(row) + @invalidRows.sort (a, b) -> a - b + @tokenizeInBackground() + + updateInvalidRows: (start, end, delta) -> + @invalidRows = @invalidRows.map (row) -> + if row < start + row + else if start <= row <= end + end + delta + 1 + else if row > end + row + delta + buildPlaceholderScreenLinesForRows: (startRow, endRow) -> @buildPlaceholderScreenLineForRow(row) for row in [startRow..endRow] @@ -188,29 +208,6 @@ class TokenizedBuffer getLastRow: -> @buffer.getLastRow() - firstInvalidRow: -> - @invalidRows[0] - - invalidateRow: (row) -> - @invalidRows.push(row) - @invalidRows.sort (a, b) -> a - b - @tokenizeInBackground() - - validateRow: (row) -> - @invalidRows.shift() if @invalidRows[0] == row - - updateInvalidRows: (start, end, delta) -> - updatedRows = [] - for row in @invalidRows - if row < start - updatedRows.push(row) - else if start <= row <= end - updatedRows.push(end + delta + 1) - else if row > end - updatedRows.push(row + delta) - - @invalidRows = updatedRows - logLines: (start=0, end=@buffer.getLastRow()) -> for row in [start..end] line = @lineForScreenRow(row).text From 7dc1490dee909801dddcab6ccdee7aaa1aa36c0f Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Fri, 23 Nov 2012 10:46:19 -0700 Subject: [PATCH 12/16] Fix stack overflow in `DisplayBuffer` spec From f466a2eede7ae9dd1077773348a446d2bede7053 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Fri, 23 Nov 2012 10:46:26 -0700 Subject: [PATCH 13/16] Un-F --- spec/app/tokenized-buffer-spec.coffee | 2 +- src/app/tokenized-buffer.coffee | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/spec/app/tokenized-buffer-spec.coffee b/spec/app/tokenized-buffer-spec.coffee index 91346de85..aaf08ed71 100644 --- a/spec/app/tokenized-buffer-spec.coffee +++ b/spec/app/tokenized-buffer-spec.coffee @@ -4,7 +4,7 @@ Buffer = require 'buffer' Range = require 'range' _ = require 'underscore' -fdescribe "TokenizedBuffer", -> +describe "TokenizedBuffer", -> [editSession, tokenizedBuffer, buffer, changeHandler] = [] beforeEach -> diff --git a/src/app/tokenized-buffer.coffee b/src/app/tokenized-buffer.coffee index 510223eb0..20e213d5f 100644 --- a/src/app/tokenized-buffer.coffee +++ b/src/app/tokenized-buffer.coffee @@ -65,7 +65,7 @@ class TokenizedBuffer tokenizeNextChunk: -> rowsRemaining = @chunkSize - while @invalidRows.length and rowsRemaining > 0 + while @firstInvalidRow()? and rowsRemaining > 0 invalidRow = @invalidRows.shift() lastRow = @getLastRow() continue if invalidRow > lastRow @@ -86,7 +86,7 @@ class TokenizedBuffer @invalidateRow(row + 1) unless filledRegion @trigger "change", { start: invalidRow, end: row, delta: 0 } - @tokenizeInBackground() + @tokenizeInBackground() if @firstInvalidRow()? firstInvalidRow: -> @invalidRows[0] From 39863e4ebe5803a74a9750122818fbabc68614fd Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Fri, 23 Nov 2012 10:48:00 -0700 Subject: [PATCH 14/16] :lipstick: --- spec/app/display-buffer-spec.coffee | 1 - 1 file changed, 1 deletion(-) diff --git a/spec/app/display-buffer-spec.coffee b/spec/app/display-buffer-spec.coffee index 1443b2d53..fa827ef74 100644 --- a/spec/app/display-buffer-spec.coffee +++ b/spec/app/display-buffer-spec.coffee @@ -565,4 +565,3 @@ describe "DisplayBuffer", -> describe ".maxLineLength()", -> it "returns the length of the longest screen line", -> expect(displayBuffer.maxLineLength()).toBe 65 - From f28cedea7dcbe9aaa7b0cc867282dab2f7e492c1 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Fri, 23 Nov 2012 12:09:29 -0700 Subject: [PATCH 15/16] :lipstick: --- src/app/tokenized-buffer.coffee | 42 ++++++++++++++++----------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/src/app/tokenized-buffer.coffee b/src/app/tokenized-buffer.coffee index 20e213d5f..bed53f2c9 100644 --- a/src/app/tokenized-buffer.coffee +++ b/src/app/tokenized-buffer.coffee @@ -25,27 +25,6 @@ class TokenizedBuffer @invalidateRow(0) @buffer.on "change.tokenized-buffer#{@id}", (e) => @handleBufferChange(e) - handleBufferChange: (e) -> - {oldRange, newRange} = e - start = oldRange.start.row - end = oldRange.end.row - delta = newRange.end.row - oldRange.end.row - - @updateInvalidRows(start, end, delta) - - previousStack = @stackForRow(end) # used in spill detection below - - stack = @stackForRow(start - 1) - if stack? or start == 0 - @screenLines[start..end] = @buildTokenizedScreenLinesForRows(start, end + delta, stack) - else - @screenLines[start..end] = @buildPlaceholderScreenLinesForRows(start, end + delta, stack) - - unless _.isEqual(@stackForRow(end + delta), previousStack) - @invalidateRow(end + delta + 1) - - @trigger "change", { start, end, delta, bufferChange: e } - getTabLength: -> @tabLength @@ -108,6 +87,27 @@ class TokenizedBuffer else if row > end row + delta + handleBufferChange: (e) -> + {oldRange, newRange} = e + start = oldRange.start.row + end = oldRange.end.row + delta = newRange.end.row - oldRange.end.row + + @updateInvalidRows(start, end, delta) + + previousStack = @stackForRow(end) # used in spill detection below + + stack = @stackForRow(start - 1) + if stack? or start == 0 + @screenLines[start..end] = @buildTokenizedScreenLinesForRows(start, end + delta, stack) + else + @screenLines[start..end] = @buildPlaceholderScreenLinesForRows(start, end + delta, stack) + + unless _.isEqual(@stackForRow(end + delta), previousStack) + @invalidateRow(end + delta + 1) + + @trigger "change", { start, end, delta, bufferChange: e } + buildPlaceholderScreenLinesForRows: (startRow, endRow) -> @buildPlaceholderScreenLineForRow(row) for row in [startRow..endRow] From a1ae8199080b34f467dcf7f898afe7642fc5e386 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Fri, 23 Nov 2012 12:14:46 -0700 Subject: [PATCH 16/16] Only tokenize in background if when a buffer is visible in an editor This will prevent running a bunch of background tokenization when Atom is reloaded. We only perform tokenization to support content that is actually on screen. --- spec/app/tokenized-buffer-spec.coffee | 2 ++ src/app/display-buffer.coffee | 2 ++ src/app/edit-session.coffee | 2 ++ src/app/editor.coffee | 1 + src/app/tokenized-buffer.coffee | 6 +++++- 5 files changed, 12 insertions(+), 1 deletion(-) diff --git a/spec/app/tokenized-buffer-spec.coffee b/spec/app/tokenized-buffer-spec.coffee index aaf08ed71..a874e43f0 100644 --- a/spec/app/tokenized-buffer-spec.coffee +++ b/spec/app/tokenized-buffer-spec.coffee @@ -21,6 +21,7 @@ describe "TokenizedBuffer", -> editSession = fixturesProject.buildEditSessionForPath('sample.js', autoIndent: false) buffer = editSession.buffer tokenizedBuffer = editSession.displayBuffer.tokenizedBuffer + editSession.setVisible(true) changeHandler = jasmine.createSpy('changeHandler') tokenizedBuffer.on "change", changeHandler @@ -289,6 +290,7 @@ describe "TokenizedBuffer", -> editSession = fixturesProject.buildEditSessionForPath('sample-with-tabs.coffee', { tabLength }) buffer = editSession.buffer tokenizedBuffer = editSession.displayBuffer.tokenizedBuffer + editSession.setVisible(true) afterEach -> editSession.destroy() diff --git a/src/app/display-buffer.coffee b/src/app/display-buffer.coffee index 568e59ebe..adf4cfd8a 100644 --- a/src/app/display-buffer.coffee +++ b/src/app/display-buffer.coffee @@ -27,6 +27,8 @@ class DisplayBuffer @buildLineMap() @tokenizedBuffer.on 'change', (e) => @handleTokenizedBufferChange(e) + setVisible: (visible) -> @tokenizedBuffer.setVisible(visible) + buildLineMap: -> @lineMap = new LineMap @lineMap.insertAtScreenRow 0, @buildLinesForBufferRows(0, @buffer.getLastRow()) diff --git a/src/app/edit-session.coffee b/src/app/edit-session.coffee index 6e88924e1..e4b5a42b7 100644 --- a/src/app/edit-session.coffee +++ b/src/app/edit-session.coffee @@ -91,6 +91,8 @@ class EditSession @scrollLeft == other.getScrollLeft() and @getCursorScreenPosition().isEqual(other.getCursorScreenPosition()) + setVisible: (visible) -> @displayBuffer.setVisible(visible) + setScrollTop: (@scrollTop) -> getScrollTop: -> @scrollTop diff --git a/src/app/editor.coffee b/src/app/editor.coffee index adb2592fa..0148be667 100644 --- a/src/app/editor.coffee +++ b/src/app/editor.coffee @@ -422,6 +422,7 @@ class Editor extends View @activeEditSession.off() @activeEditSession = @editSessions[index] + @activeEditSession.setVisible(true) @activeEditSession.on "buffer-contents-change-on-disk", => @showBufferConflictAlert(@activeEditSession) diff --git a/src/app/tokenized-buffer.coffee b/src/app/tokenized-buffer.coffee index bed53f2c9..d1c0cb742 100644 --- a/src/app/tokenized-buffer.coffee +++ b/src/app/tokenized-buffer.coffee @@ -16,6 +16,7 @@ class TokenizedBuffer screenLines: null chunkSize: 50 invalidRows: null + visible: false constructor: (@buffer, { @languageMode, @tabLength }) -> @tabLength ?= 2 @@ -25,6 +26,9 @@ class TokenizedBuffer @invalidateRow(0) @buffer.on "change.tokenized-buffer#{@id}", (e) => @handleBufferChange(e) + setVisible: (@visible) -> + @tokenizeInBackground() if @visible + getTabLength: -> @tabLength @@ -35,7 +39,7 @@ class TokenizedBuffer @trigger "change", { start: 0, end: lastRow, delta: 0 } tokenizeInBackground: -> - return if @pendingChunk + return if not @visible or @pendingChunk @pendingChunk = true _.defer => @pendingChunk = false