diff --git a/spec/app/tokenized-buffer-spec.coffee b/spec/app/tokenized-buffer-spec.coffee index 32dd9c8c1..b7532e9a3 100644 --- a/spec/app/tokenized-buffer-spec.coffee +++ b/spec/app/tokenized-buffer-spec.coffee @@ -13,7 +13,7 @@ fdescribe "TokenizedBuffer", -> jasmine.unspy(TokenizedBuffer.prototype, 'tokenizeInBackground') fullyTokenize = (tokenizedBuffer) -> - advanceClock() while tokenizedBuffer.untokenizedRow <= tokenizedBuffer.getLastRow() + advanceClock() while tokenizedBuffer.firstInvalidRow()? changeHandler.reset() describe "when the buffer contains soft-tabs", -> @@ -76,6 +76,7 @@ fdescribe "TokenizedBuffer", -> describe "when the buffer is fully tokenized", -> beforeEach -> + console.log "FULLY TOKENIZE" fullyTokenize(tokenizedBuffer) describe "when there is a buffer change that is smaller than the chunk size", -> @@ -122,9 +123,8 @@ fdescribe "TokenizedBuffer", -> expect(tokenizedBuffer.lineForScreenRow(1).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] describe "when lines are both updated and removed", -> - it "updates tokens to reflect the removed lines", -> - range = new Range([1, 0], [3, 0]) - buffer.change(range, "foo()") + it "updates tokens to reflect the change", -> + buffer.change([[1, 0], [3, 0]], "foo()") # previous line 0 remains expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.modifier.js']) @@ -143,24 +143,31 @@ fdescribe "TokenizedBuffer", -> delete event.bufferChange expect(event).toEqual(start: 1, end: 3, delta: -2) - it "updates tokens for lines beyond the changed lines if needed", -> + describe "when the change invalidates the tokenization of subsequent lines", -> + it "schedules the invalidated lines to be tokenized in the background", -> buffer.insert([5, 30], '/* */') changeHandler.reset() - buffer.change(new Range([2, 0], [3, 0]), '/*') + buffer.change([[2, 0], [3, 0]], '/*') expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] - expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - + expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js'] expect(changeHandler).toHaveBeenCalled() [event] = changeHandler.argsForCall[0] delete event.bufferChange - expect(event).toEqual(start: 2, end: 5, delta: -1) + expect(event).toEqual(start: 2, end: 3, delta: -1) + changeHandler.reset() + + advanceClock() + expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(changeHandler).toHaveBeenCalled() + [event] = changeHandler.argsForCall[0] + delete event.bufferChange + expect(event).toEqual(start: 3, end: 4, delta: 0) describe "when lines are both updated and inserted", -> - it "updates tokens to reflect the inserted lines", -> - range = new Range([1, 0], [2, 0]) - buffer.change(range, "foo()\nbar()\nbaz()\nquux()") + it "updates tokens to reflect the change", -> + buffer.change([[1, 0], [2, 0]], "foo()\nbar()\nbaz()\nquux()") # previous line 0 remains expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual( value: 'var', scopes: ['source.js', 'storage.modifier.js']) @@ -182,14 +189,23 @@ fdescribe "TokenizedBuffer", -> delete event.bufferChange expect(event).toEqual(start: 1, end: 2, delta: 2) - it "updates tokens for lines beyond the changed lines if needed", -> + describe "when the change invalidates the tokenization of subsequent lines", -> + it "schedules the invalidated lines to be tokenized in the background", -> buffer.insert([5, 30], '/* */') changeHandler.reset() buffer.insert([2, 0], '/*\nabcde\nabcder') + expect(changeHandler).toHaveBeenCalled() + [event] = changeHandler.argsForCall[0] + delete event.bufferChange + expect(event).toEqual(start: 2, end: 2, delta: 2) expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js'] + changeHandler.reset() + + advanceClock() # tokenize invalidated lines in background expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] expect(tokenizedBuffer.lineForScreenRow(6).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] expect(tokenizedBuffer.lineForScreenRow(7).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] @@ -198,7 +214,7 @@ fdescribe "TokenizedBuffer", -> expect(changeHandler).toHaveBeenCalled() [event] = changeHandler.argsForCall[0] delete event.bufferChange - expect(event).toEqual(start: 2, end: 5, delta: 2) + expect(event).toEqual(start: 5, end: 7, delta: 0) describe ".findOpeningBracket(closingBufferPosition)", -> it "returns the position of the matching bracket, skipping any nested brackets", -> diff --git a/src/app/tokenized-buffer.coffee b/src/app/tokenized-buffer.coffee index f2e3a171b..53647ba2a 100644 --- a/src/app/tokenized-buffer.coffee +++ b/src/app/tokenized-buffer.coffee @@ -21,8 +21,8 @@ class TokenizedBuffer @tabLength ?= 2 @id = @constructor.idCounter++ @screenLines = @buildPlaceholderScreenLinesForRows(0, @buffer.getLastRow()) - @invalidRows = [0] - @tokenizeInBackground() + @invalidRows = [] + @invalidateRow(0) @buffer.on "change.tokenized-buffer#{@id}", (e) => @handleBufferChange(e) handleBufferChange: (e) -> @@ -37,27 +37,10 @@ class TokenizedBuffer @screenLines[start..end] = @buildTokenizedScreenLinesForRows(start, end + delta, stack) - # spill detection - # compare scanner state of last re-highlighted line with its previous state. - # if it differs, re-tokenize the next line with the new state and repeat for - # each line until the line's new state matches the previous state. this covers - # cases like inserting a /* needing to comment out lines below until we see a */ - - unless _.isEqual(@stackForRow(end + delta), previousStack) console.log "spill" - @invalidRows.unshift(end + 1) - @tokenizeInBackground() + @invalidateRow(end + delta + 1) -# for row in [(end + delta)...@buffer.getLastRow()] -# -# nextRow = row + 1 -# previousStack = @stackForRow(nextRow) -# @screenLines[nextRow] = @buildTokenizedScreenLineForRow(nextRow, @stackForRow(row)) - - # if highlighting spilled beyond the bounds of the textual change, update the - # end of the affected range to reflect the larger area of highlighting -# end = Math.max(end, nextRow - delta) if nextRow @trigger "change", { start, end, delta, bufferChange: e } getTabLength: -> @@ -65,9 +48,8 @@ class TokenizedBuffer setTabLength: (@tabLength) -> lastRow = @buffer.getLastRow() - @invalidRows = [0] @screenLines = @buildPlaceholderScreenLinesForRows(0, lastRow) - @tokenizeInBackground() + @invalidateRow(0) @trigger "change", { start: 0, end: lastRow, delta: 0 } tokenizeInBackground: -> @@ -90,15 +72,15 @@ class TokenizedBuffer loop previousStack = @stackForRow(row) @screenLines[row] = @buildTokenizedScreenLineForRow(row, @stackForRow(row - 1)) + if --rowsRemaining == 0 + break if row == lastRow or _.isEqual(@stackForRow(row), previousStack) filledRegion = true break - if --rowsRemaining == 0 - break row++ @trigger "change", { start: invalidRow, end: row, delta: 0} - @invalidRows.unshift(row + 1) unless filledRegion + @invalidateRow(row + 1) unless filledRegion @tokenizeInBackground() @@ -202,6 +184,14 @@ class TokenizedBuffer getLastRow: -> @buffer.getLastRow() + firstInvalidRow: -> + @invalidRows[0] + + invalidateRow: (row) -> + @invalidRows.push(row) + @invalidRows.sort() + @tokenizeInBackground() + logLines: (start=0, end=@buffer.getLastRow()) -> for row in [start..end] line = @lineForScreenRow(row).text