mirror of
https://github.com/atom/atom.git
synced 2026-04-28 03:01:47 -04:00
Merge pull request #99 from github/async-tokenization
Tokenize asynchronously
This commit is contained in:
Nathan Sobo
@@ -565,4 +565,3 @@ describe "DisplayBuffer", ->
|
||||
describe ".maxLineLength()", ->
|
||||
it "returns the length of the longest screen line", ->
|
||||
expect(displayBuffer.maxLineLength()).toBe 65
|
||||
|
||||
|
||||
@@ -5,166 +5,302 @@ Range = require 'range'
|
||||
_ = require 'underscore'
|
||||
|
||||
describe "TokenizedBuffer", ->
|
||||
[editSession, tokenizedBuffer, buffer] = []
|
||||
[editSession, tokenizedBuffer, buffer, changeHandler] = []
|
||||
|
||||
beforeEach ->
|
||||
editSession = fixturesProject.buildEditSessionForPath('sample.js', autoIndent: false)
|
||||
buffer = editSession.buffer
|
||||
tokenizedBuffer = editSession.displayBuffer.tokenizedBuffer
|
||||
# enable async tokenization
|
||||
TokenizedBuffer.prototype.chunkSize = 5
|
||||
jasmine.unspy(TokenizedBuffer.prototype, 'tokenizeInBackground')
|
||||
|
||||
afterEach ->
|
||||
editSession.destroy()
|
||||
fullyTokenize = (tokenizedBuffer) ->
|
||||
advanceClock() while tokenizedBuffer.firstInvalidRow()?
|
||||
changeHandler.reset()
|
||||
|
||||
describe ".findOpeningBracket(closingBufferPosition)", ->
|
||||
it "returns the position of the matching bracket, skipping any nested brackets", ->
|
||||
expect(tokenizedBuffer.findOpeningBracket([9, 2])).toEqual [1, 29]
|
||||
describe "when the buffer contains soft-tabs", ->
|
||||
beforeEach ->
|
||||
editSession = fixturesProject.buildEditSessionForPath('sample.js', autoIndent: false)
|
||||
buffer = editSession.buffer
|
||||
tokenizedBuffer = editSession.displayBuffer.tokenizedBuffer
|
||||
editSession.setVisible(true)
|
||||
changeHandler = jasmine.createSpy('changeHandler')
|
||||
tokenizedBuffer.on "change", changeHandler
|
||||
|
||||
describe ".findClosingBracket(startBufferPosition)", ->
|
||||
it "returns the position of the matching bracket, skipping any nested brackets", ->
|
||||
expect(tokenizedBuffer.findClosingBracket([1, 29])).toEqual [9, 2]
|
||||
afterEach ->
|
||||
editSession.destroy()
|
||||
|
||||
describe "tokenization", ->
|
||||
it "tokenizes all the lines in the buffer on construction", ->
|
||||
expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.modifier.js'])
|
||||
expect(tokenizedBuffer.lineForScreenRow(11).tokens[1]).toEqual(value: 'return', scopes: ['source.js', 'keyword.control.js'])
|
||||
describe "on construction", ->
|
||||
it "initially creates un-tokenized screen lines, then tokenizes lines chunk at a time in the background", ->
|
||||
line0 = tokenizedBuffer.lineForScreenRow(0)
|
||||
expect(line0.tokens.length).toBe 1
|
||||
expect(line0.tokens[0]).toEqual(value: line0.text, scopes: ['source.js'])
|
||||
|
||||
describe "when the buffer changes", ->
|
||||
changeHandler = null
|
||||
line11 = tokenizedBuffer.lineForScreenRow(11)
|
||||
expect(line11.tokens.length).toBe 2
|
||||
expect(line11.tokens[0]).toEqual(value: " ", scopes: ['source.js'], isAtomic: true)
|
||||
expect(line11.tokens[1]).toEqual(value: "return sort(Array.apply(this, arguments));", scopes: ['source.js'])
|
||||
|
||||
# background tokenization has not begun
|
||||
expect(tokenizedBuffer.lineForScreenRow(0).ruleStack).toBeUndefined()
|
||||
|
||||
# tokenize chunk 1
|
||||
advanceClock()
|
||||
expect(tokenizedBuffer.lineForScreenRow(0).ruleStack?).toBeTruthy()
|
||||
expect(tokenizedBuffer.lineForScreenRow(4).ruleStack?).toBeTruthy()
|
||||
expect(tokenizedBuffer.lineForScreenRow(5).ruleStack?).toBeFalsy()
|
||||
expect(changeHandler).toHaveBeenCalledWith(start: 0, end: 4, delta: 0)
|
||||
changeHandler.reset()
|
||||
|
||||
# tokenize chunk 2
|
||||
advanceClock()
|
||||
expect(tokenizedBuffer.lineForScreenRow(5).ruleStack?).toBeTruthy()
|
||||
expect(tokenizedBuffer.lineForScreenRow(9).ruleStack?).toBeTruthy()
|
||||
expect(tokenizedBuffer.lineForScreenRow(10).ruleStack?).toBeFalsy()
|
||||
expect(changeHandler).toHaveBeenCalledWith(start: 5, end: 9, delta: 0)
|
||||
changeHandler.reset()
|
||||
|
||||
# tokenize last chunk
|
||||
advanceClock()
|
||||
expect(tokenizedBuffer.lineForScreenRow(10).ruleStack?).toBeTruthy()
|
||||
expect(tokenizedBuffer.lineForScreenRow(12).ruleStack?).toBeTruthy()
|
||||
expect(changeHandler).toHaveBeenCalledWith(start: 10, end: 12, delta: 0)
|
||||
|
||||
describe "when the buffer is partially tokenized", ->
|
||||
beforeEach ->
|
||||
changeHandler = jasmine.createSpy('changeHandler')
|
||||
tokenizedBuffer.on "change", changeHandler
|
||||
# tokenize chunk 1 only
|
||||
advanceClock()
|
||||
changeHandler.reset()
|
||||
|
||||
describe "when lines are updated, but none are added or removed", ->
|
||||
it "updates tokens for each of the changed lines", ->
|
||||
range = new Range([0, 0], [2, 0])
|
||||
buffer.change(range, "foo()\n7\n")
|
||||
describe "when there is a buffer change inside the tokenized region", ->
|
||||
describe "when lines are added", ->
|
||||
it "pushes the invalid rows down", ->
|
||||
expect(tokenizedBuffer.firstInvalidRow()).toBe 5
|
||||
buffer.insert([1, 0], '\n\n')
|
||||
changeHandler.reset()
|
||||
|
||||
expect(tokenizedBuffer.lineForScreenRow(0).tokens[1]).toEqual(value: '(', scopes: ['source.js', 'meta.brace.round.js'])
|
||||
expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: '7', scopes: ['source.js', 'constant.numeric.js'])
|
||||
# line 2 is unchanged
|
||||
expect(tokenizedBuffer.lineForScreenRow(2).tokens[2]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js'])
|
||||
expect(tokenizedBuffer.firstInvalidRow()).toBe 7
|
||||
advanceClock()
|
||||
expect(changeHandler).toHaveBeenCalledWith(start: 7, end: 11, delta: 0)
|
||||
|
||||
expect(changeHandler).toHaveBeenCalled()
|
||||
[event] = changeHandler.argsForCall[0]
|
||||
delete event.bufferChange
|
||||
expect(event).toEqual(start: 0, end: 2, delta: 0)
|
||||
describe "when lines are removed", ->
|
||||
it "pulls the invalid rows up", ->
|
||||
expect(tokenizedBuffer.firstInvalidRow()).toBe 5
|
||||
buffer.delete([[1, 0], [3, 0]])
|
||||
changeHandler.reset()
|
||||
|
||||
it "updates tokens for lines beyond the changed lines if needed", ->
|
||||
buffer.insert([5, 30], '/* */')
|
||||
expect(tokenizedBuffer.firstInvalidRow()).toBe 3
|
||||
advanceClock()
|
||||
expect(changeHandler).toHaveBeenCalledWith(start: 3, end: 7, delta: 0)
|
||||
|
||||
describe "when the change invalidates all the lines before the current invalid region", ->
|
||||
it "retokenizes the invalidated lines and continues into the valid region", ->
|
||||
expect(tokenizedBuffer.firstInvalidRow()).toBe 5
|
||||
buffer.insert([2, 0], '/*')
|
||||
changeHandler.reset()
|
||||
expect(tokenizedBuffer.firstInvalidRow()).toBe 3
|
||||
|
||||
advanceClock()
|
||||
expect(changeHandler).toHaveBeenCalledWith(start: 3, end: 7, delta: 0)
|
||||
expect(tokenizedBuffer.firstInvalidRow()).toBe 8
|
||||
|
||||
describe "when there is a buffer change surrounding an invalid row", ->
|
||||
it "pushes the invalid row to the end of the change", ->
|
||||
buffer.change([[4, 0], [6, 0]], "\n\n\n")
|
||||
changeHandler.reset()
|
||||
|
||||
buffer.insert([2, 0], '/*')
|
||||
expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
expect(tokenizedBuffer.firstInvalidRow()).toBe 8
|
||||
advanceClock()
|
||||
|
||||
expect(changeHandler).toHaveBeenCalled()
|
||||
[event] = changeHandler.argsForCall[0]
|
||||
delete event.bufferChange
|
||||
expect(event).toEqual(start: 2, end: 5, delta: 0)
|
||||
describe "when there is a buffer change inside an invalid region", ->
|
||||
it "does not attempt to tokenize the lines in the change, and preserves the existing invalid row", ->
|
||||
expect(tokenizedBuffer.firstInvalidRow()).toBe 5
|
||||
buffer.change([[6, 0], [7, 0]], "\n\n\n")
|
||||
|
||||
it "resumes highlighting with the state of the previous line", ->
|
||||
buffer.insert([0, 0], '/*')
|
||||
buffer.insert([5, 0], '*/')
|
||||
expect(tokenizedBuffer.lineForScreenRow(6).ruleStack?).toBeFalsy()
|
||||
expect(tokenizedBuffer.lineForScreenRow(7).ruleStack?).toBeFalsy()
|
||||
|
||||
buffer.insert([1, 0], 'var ')
|
||||
expect(tokenizedBuffer.lineForScreenRow(1).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
|
||||
describe "when lines are both updated and removed", ->
|
||||
it "updates tokens to reflect the removed lines", ->
|
||||
range = new Range([1, 0], [3, 0])
|
||||
buffer.change(range, "foo()")
|
||||
|
||||
# previous line 0 remains
|
||||
expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.modifier.js'])
|
||||
|
||||
# previous line 3 should be combined with input to form line 1
|
||||
expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js'])
|
||||
expect(tokenizedBuffer.lineForScreenRow(1).tokens[6]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
|
||||
|
||||
# lines below deleted regions should be shifted upward
|
||||
expect(tokenizedBuffer.lineForScreenRow(2).tokens[2]).toEqual(value: 'while', scopes: ['source.js', 'keyword.control.js'])
|
||||
expect(tokenizedBuffer.lineForScreenRow(3).tokens[4]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
|
||||
expect(tokenizedBuffer.lineForScreenRow(4).tokens[4]).toEqual(value: '<', scopes: ['source.js', 'keyword.operator.js'])
|
||||
|
||||
expect(changeHandler).toHaveBeenCalled()
|
||||
[event] = changeHandler.argsForCall[0]
|
||||
delete event.bufferChange
|
||||
expect(event).toEqual(start: 1, end: 3, delta: -2)
|
||||
|
||||
it "updates tokens for lines beyond the changed lines if needed", ->
|
||||
buffer.insert([5, 30], '/* */')
|
||||
changeHandler.reset()
|
||||
expect(tokenizedBuffer.firstInvalidRow()).toBe 5
|
||||
|
||||
buffer.change(new Range([2, 0], [3, 0]), '/*')
|
||||
expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
|
||||
expect(changeHandler).toHaveBeenCalled()
|
||||
[event] = changeHandler.argsForCall[0]
|
||||
delete event.bufferChange
|
||||
expect(event).toEqual(start: 2, end: 5, delta: -1)
|
||||
|
||||
describe "when lines are both updated and inserted", ->
|
||||
it "updates tokens to reflect the inserted lines", ->
|
||||
range = new Range([1, 0], [2, 0])
|
||||
buffer.change(range, "foo()\nbar()\nbaz()\nquux()")
|
||||
|
||||
# previous line 0 remains
|
||||
expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual( value: 'var', scopes: ['source.js', 'storage.modifier.js'])
|
||||
|
||||
# 3 new lines inserted
|
||||
expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js'])
|
||||
expect(tokenizedBuffer.lineForScreenRow(2).tokens[0]).toEqual(value: 'bar', scopes: ['source.js'])
|
||||
expect(tokenizedBuffer.lineForScreenRow(3).tokens[0]).toEqual(value: 'baz', scopes: ['source.js'])
|
||||
|
||||
# previous line 2 is joined with quux() on line 4
|
||||
expect(tokenizedBuffer.lineForScreenRow(4).tokens[0]).toEqual(value: 'quux', scopes: ['source.js'])
|
||||
expect(tokenizedBuffer.lineForScreenRow(4).tokens[4]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js'])
|
||||
|
||||
# previous line 3 is pushed down to become line 5
|
||||
expect(tokenizedBuffer.lineForScreenRow(5).tokens[4]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
|
||||
|
||||
expect(changeHandler).toHaveBeenCalled()
|
||||
[event] = changeHandler.argsForCall[0]
|
||||
delete event.bufferChange
|
||||
expect(event).toEqual(start: 1, end: 2, delta: 2)
|
||||
|
||||
it "updates tokens for lines beyond the changed lines if needed", ->
|
||||
buffer.insert([5, 30], '/* */')
|
||||
changeHandler.reset()
|
||||
|
||||
buffer.insert([2, 0], '/*\nabcde\nabcder')
|
||||
expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(6).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(7).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(8).tokens[0].scopes).not.toBe ['source.js', 'comment.block.js']
|
||||
|
||||
expect(changeHandler).toHaveBeenCalled()
|
||||
[event] = changeHandler.argsForCall[0]
|
||||
delete event.bufferChange
|
||||
expect(event).toEqual(start: 2, end: 5, delta: 2)
|
||||
|
||||
|
||||
describe "when the buffer contains tab characters", ->
|
||||
editSession2 = null
|
||||
|
||||
describe "when the buffer is fully tokenized", ->
|
||||
beforeEach ->
|
||||
tabLength = 2
|
||||
editSession2 = fixturesProject.buildEditSessionForPath('sample-with-tabs.coffee', { tabLength })
|
||||
buffer = editSession2.buffer
|
||||
tokenizedBuffer = editSession2.displayBuffer.tokenizedBuffer
|
||||
fullyTokenize(tokenizedBuffer)
|
||||
|
||||
afterEach ->
|
||||
editSession2.destroy()
|
||||
describe "when there is a buffer change that is smaller than the chunk size", ->
|
||||
describe "when lines are updated, but none are added or removed", ->
|
||||
it "updates tokens to reflect the change", ->
|
||||
buffer.change([[0, 0], [2, 0]], "foo()\n7\n")
|
||||
|
||||
it "always renders each tab as its own atomic token with a value of size tabLength", ->
|
||||
tabAsSpaces = _.multiplyString(' ', editSession2.getTabLength())
|
||||
expect(tokenizedBuffer.lineForScreenRow(0).tokens[1]).toEqual(value: '(', scopes: ['source.js', 'meta.brace.round.js'])
|
||||
expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: '7', scopes: ['source.js', 'constant.numeric.js'])
|
||||
# line 2 is unchanged
|
||||
expect(tokenizedBuffer.lineForScreenRow(2).tokens[2]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js'])
|
||||
|
||||
expect(changeHandler).toHaveBeenCalled()
|
||||
[event] = changeHandler.argsForCall[0]
|
||||
delete event.bufferChange
|
||||
expect(event).toEqual(start: 0, end: 2, delta: 0)
|
||||
|
||||
describe "when the change invalidates the tokenization of subsequent lines", ->
|
||||
it "schedules the invalidated lines to be tokenized in the background", ->
|
||||
buffer.insert([5, 30], '/* */')
|
||||
changeHandler.reset()
|
||||
buffer.insert([2, 0], '/*')
|
||||
expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js']
|
||||
expect(changeHandler).toHaveBeenCalled()
|
||||
[event] = changeHandler.argsForCall[0]
|
||||
delete event.bufferChange
|
||||
expect(event).toEqual(start: 2, end: 2, delta: 0)
|
||||
changeHandler.reset()
|
||||
|
||||
advanceClock()
|
||||
expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
expect(changeHandler).toHaveBeenCalled()
|
||||
[event] = changeHandler.argsForCall[0]
|
||||
delete event.bufferChange
|
||||
expect(event).toEqual(start: 3, end: 5, delta: 0)
|
||||
|
||||
it "resumes highlighting with the state of the previous line", ->
|
||||
buffer.insert([0, 0], '/*')
|
||||
buffer.insert([5, 0], '*/')
|
||||
|
||||
buffer.insert([1, 0], 'var ')
|
||||
expect(tokenizedBuffer.lineForScreenRow(1).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
|
||||
describe "when lines are both updated and removed", ->
|
||||
it "updates tokens to reflect the change", ->
|
||||
buffer.change([[1, 0], [3, 0]], "foo()")
|
||||
|
||||
# previous line 0 remains
|
||||
expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.modifier.js'])
|
||||
|
||||
# previous line 3 should be combined with input to form line 1
|
||||
expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js'])
|
||||
expect(tokenizedBuffer.lineForScreenRow(1).tokens[6]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
|
||||
|
||||
# lines below deleted regions should be shifted upward
|
||||
expect(tokenizedBuffer.lineForScreenRow(2).tokens[2]).toEqual(value: 'while', scopes: ['source.js', 'keyword.control.js'])
|
||||
expect(tokenizedBuffer.lineForScreenRow(3).tokens[4]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
|
||||
expect(tokenizedBuffer.lineForScreenRow(4).tokens[4]).toEqual(value: '<', scopes: ['source.js', 'keyword.operator.js'])
|
||||
|
||||
expect(changeHandler).toHaveBeenCalled()
|
||||
[event] = changeHandler.argsForCall[0]
|
||||
delete event.bufferChange
|
||||
expect(event).toEqual(start: 1, end: 3, delta: -2)
|
||||
|
||||
describe "when the change invalidates the tokenization of subsequent lines", ->
|
||||
it "schedules the invalidated lines to be tokenized in the background", ->
|
||||
buffer.insert([5, 30], '/* */')
|
||||
changeHandler.reset()
|
||||
|
||||
buffer.change([[2, 0], [3, 0]], '/*')
|
||||
expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js']
|
||||
expect(changeHandler).toHaveBeenCalled()
|
||||
[event] = changeHandler.argsForCall[0]
|
||||
delete event.bufferChange
|
||||
expect(event).toEqual(start: 2, end: 3, delta: -1)
|
||||
changeHandler.reset()
|
||||
|
||||
advanceClock()
|
||||
expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
expect(changeHandler).toHaveBeenCalled()
|
||||
[event] = changeHandler.argsForCall[0]
|
||||
delete event.bufferChange
|
||||
expect(event).toEqual(start: 3, end: 4, delta: 0)
|
||||
|
||||
describe "when lines are both updated and inserted", ->
|
||||
it "updates tokens to reflect the change", ->
|
||||
buffer.change([[1, 0], [2, 0]], "foo()\nbar()\nbaz()\nquux()")
|
||||
|
||||
# previous line 0 remains
|
||||
expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual( value: 'var', scopes: ['source.js', 'storage.modifier.js'])
|
||||
|
||||
# 3 new lines inserted
|
||||
expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js'])
|
||||
expect(tokenizedBuffer.lineForScreenRow(2).tokens[0]).toEqual(value: 'bar', scopes: ['source.js'])
|
||||
expect(tokenizedBuffer.lineForScreenRow(3).tokens[0]).toEqual(value: 'baz', scopes: ['source.js'])
|
||||
|
||||
# previous line 2 is joined with quux() on line 4
|
||||
expect(tokenizedBuffer.lineForScreenRow(4).tokens[0]).toEqual(value: 'quux', scopes: ['source.js'])
|
||||
expect(tokenizedBuffer.lineForScreenRow(4).tokens[4]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js'])
|
||||
|
||||
# previous line 3 is pushed down to become line 5
|
||||
expect(tokenizedBuffer.lineForScreenRow(5).tokens[4]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
|
||||
|
||||
expect(changeHandler).toHaveBeenCalled()
|
||||
[event] = changeHandler.argsForCall[0]
|
||||
delete event.bufferChange
|
||||
expect(event).toEqual(start: 1, end: 2, delta: 2)
|
||||
|
||||
describe "when the change invalidates the tokenization of subsequent lines", ->
|
||||
it "schedules the invalidated lines to be tokenized in the background", ->
|
||||
buffer.insert([5, 30], '/* */')
|
||||
changeHandler.reset()
|
||||
|
||||
buffer.insert([2, 0], '/*\nabcde\nabcder')
|
||||
expect(changeHandler).toHaveBeenCalled()
|
||||
[event] = changeHandler.argsForCall[0]
|
||||
delete event.bufferChange
|
||||
expect(event).toEqual(start: 2, end: 2, delta: 2)
|
||||
expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js']
|
||||
changeHandler.reset()
|
||||
|
||||
advanceClock() # tokenize invalidated lines in background
|
||||
expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(6).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(7).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
|
||||
expect(tokenizedBuffer.lineForScreenRow(8).tokens[0].scopes).not.toBe ['source.js', 'comment.block.js']
|
||||
|
||||
expect(changeHandler).toHaveBeenCalled()
|
||||
[event] = changeHandler.argsForCall[0]
|
||||
delete event.bufferChange
|
||||
expect(event).toEqual(start: 5, end: 7, delta: 0)
|
||||
|
||||
describe ".findOpeningBracket(closingBufferPosition)", ->
|
||||
it "returns the position of the matching bracket, skipping any nested brackets", ->
|
||||
expect(tokenizedBuffer.findOpeningBracket([9, 2])).toEqual [1, 29]
|
||||
|
||||
describe ".findClosingBracket(startBufferPosition)", ->
|
||||
it "returns the position of the matching bracket, skipping any nested brackets", ->
|
||||
expect(tokenizedBuffer.findClosingBracket([1, 29])).toEqual [9, 2]
|
||||
|
||||
it "tokenizes leading whitespace based on the new tab length", ->
|
||||
expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].isAtomic).toBeTruthy()
|
||||
expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].value).toBe " "
|
||||
expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].isAtomic).toBeTruthy()
|
||||
expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].value).toBe " "
|
||||
|
||||
tokenizedBuffer.setTabLength(4)
|
||||
fullyTokenize(tokenizedBuffer)
|
||||
|
||||
expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].isAtomic).toBeTruthy()
|
||||
expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].value).toBe " "
|
||||
expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].isAtomic).toBeFalsy()
|
||||
expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].value).toBe " current "
|
||||
|
||||
describe "when the buffer contains hard-tabs", ->
|
||||
beforeEach ->
|
||||
tabLength = 2
|
||||
editSession = fixturesProject.buildEditSessionForPath('sample-with-tabs.coffee', { tabLength })
|
||||
buffer = editSession.buffer
|
||||
tokenizedBuffer = editSession.displayBuffer.tokenizedBuffer
|
||||
editSession.setVisible(true)
|
||||
|
||||
afterEach ->
|
||||
editSession.destroy()
|
||||
|
||||
describe "when the buffer is fully tokenized", ->
|
||||
beforeEach ->
|
||||
fullyTokenize(tokenizedBuffer)
|
||||
|
||||
it "renders each tab as its own atomic token with a value of size tabLength", ->
|
||||
tabAsSpaces = _.multiplyString(' ', editSession.getTabLength())
|
||||
screenLine0 = tokenizedBuffer.lineForScreenRow(0)
|
||||
expect(screenLine0.text).toBe "# Econ 101#{tabAsSpaces}"
|
||||
{ tokens } = screenLine0
|
||||
@@ -178,16 +314,3 @@ describe "TokenizedBuffer", ->
|
||||
|
||||
expect(tokenizedBuffer.lineForScreenRow(2).text).toBe "#{tabAsSpaces} buy()#{tabAsSpaces}while supply > demand"
|
||||
|
||||
describe ".setTabLength(tabLength)", ->
|
||||
describe "when the file contains soft tabs", ->
|
||||
it "retokenizes leading whitespace based on the new tab length", ->
|
||||
expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].isAtomic).toBeTruthy()
|
||||
expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].value).toBe " "
|
||||
expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].isAtomic).toBeTruthy()
|
||||
expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].value).toBe " "
|
||||
|
||||
tokenizedBuffer.setTabLength(4)
|
||||
expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].isAtomic).toBeTruthy()
|
||||
expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].value).toBe " "
|
||||
expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].isAtomic).toBeFalsy()
|
||||
expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].value).toBe " current "
|
||||
|
||||
@@ -10,6 +10,7 @@ RootView = require 'root-view'
|
||||
Editor = require 'editor'
|
||||
TextMateBundle = require 'text-mate-bundle'
|
||||
TextMateTheme = require 'text-mate-theme'
|
||||
TokenizedBuffer = require 'tokenized-buffer'
|
||||
fs = require 'fs'
|
||||
require 'window'
|
||||
|
||||
@@ -29,6 +30,10 @@ beforeEach ->
|
||||
spyOn(window, "clearTimeout").andCallFake window.fakeClearTimeout
|
||||
spyOn(File.prototype, "detectResurrectionAfterDelay").andCallFake -> @detectResurrection()
|
||||
|
||||
# make tokenization synchronous
|
||||
TokenizedBuffer.prototype.chunkSize = Infinity
|
||||
spyOn(TokenizedBuffer.prototype, "tokenizeInBackground").andCallFake -> @tokenizeNextChunk()
|
||||
|
||||
afterEach ->
|
||||
delete window.rootView if window.rootView
|
||||
$('#jasmine-content').empty()
|
||||
|
||||
Reference in New Issue
Block a user