lineForScreenRow -> tokenizedLineForRow

The method was severely mislabeled
This commit is contained in:
Ben Ogle
2014-09-03 13:32:23 -07:00
parent 2b9b4a48ef
commit 99f899dc4a
6 changed files with 163 additions and 166 deletions

View File

@@ -83,7 +83,7 @@ describe "Editor", ->
screenLines = []
bufferRows = []
for bufferRow in [0..tokenizedBuffer.getLastRow()]
for screenLine in softWrapLine(tokenizedBuffer.lineForScreenRow(bufferRow))
for screenLine in softWrapLine(tokenizedBuffer.tokenizedLineForRow(bufferRow))
screenLines.push(screenLine)
bufferRows.push(bufferRow)
else

View File

@@ -49,38 +49,38 @@ describe "TokenizedBuffer", ->
describe "on construction", ->
it "initially creates un-tokenized screen lines, then tokenizes lines chunk at a time in the background", ->
line0 = tokenizedBuffer.lineForScreenRow(0)
line0 = tokenizedBuffer.tokenizedLineForRow(0)
expect(line0.tokens.length).toBe 1
expect(line0.tokens[0]).toEqual(value: line0.text, scopes: ['source.js'])
line11 = tokenizedBuffer.lineForScreenRow(11)
line11 = tokenizedBuffer.tokenizedLineForRow(11)
expect(line11.tokens.length).toBe 2
expect(line11.tokens[0]).toEqual(value: " ", scopes: ['source.js'], isAtomic: true)
expect(line11.tokens[1]).toEqual(value: "return sort(Array.apply(this, arguments));", scopes: ['source.js'])
# background tokenization has not begun
expect(tokenizedBuffer.lineForScreenRow(0).ruleStack).toBeUndefined()
expect(tokenizedBuffer.tokenizedLineForRow(0).ruleStack).toBeUndefined()
# tokenize chunk 1
advanceClock()
expect(tokenizedBuffer.lineForScreenRow(0).ruleStack?).toBeTruthy()
expect(tokenizedBuffer.lineForScreenRow(4).ruleStack?).toBeTruthy()
expect(tokenizedBuffer.lineForScreenRow(5).ruleStack?).toBeFalsy()
expect(tokenizedBuffer.tokenizedLineForRow(0).ruleStack?).toBeTruthy()
expect(tokenizedBuffer.tokenizedLineForRow(4).ruleStack?).toBeTruthy()
expect(tokenizedBuffer.tokenizedLineForRow(5).ruleStack?).toBeFalsy()
expect(changeHandler).toHaveBeenCalledWith(start: 0, end: 4, delta: 0)
changeHandler.reset()
# tokenize chunk 2
advanceClock()
expect(tokenizedBuffer.lineForScreenRow(5).ruleStack?).toBeTruthy()
expect(tokenizedBuffer.lineForScreenRow(9).ruleStack?).toBeTruthy()
expect(tokenizedBuffer.lineForScreenRow(10).ruleStack?).toBeFalsy()
expect(tokenizedBuffer.tokenizedLineForRow(5).ruleStack?).toBeTruthy()
expect(tokenizedBuffer.tokenizedLineForRow(9).ruleStack?).toBeTruthy()
expect(tokenizedBuffer.tokenizedLineForRow(10).ruleStack?).toBeFalsy()
expect(changeHandler).toHaveBeenCalledWith(start: 5, end: 9, delta: 0)
changeHandler.reset()
# tokenize last chunk
advanceClock()
expect(tokenizedBuffer.lineForScreenRow(10).ruleStack?).toBeTruthy()
expect(tokenizedBuffer.lineForScreenRow(12).ruleStack?).toBeTruthy()
expect(tokenizedBuffer.tokenizedLineForRow(10).ruleStack?).toBeTruthy()
expect(tokenizedBuffer.tokenizedLineForRow(12).ruleStack?).toBeTruthy()
expect(changeHandler).toHaveBeenCalledWith(start: 10, end: 12, delta: 0)
describe "when the buffer is partially tokenized", ->
@@ -134,8 +134,8 @@ describe "TokenizedBuffer", ->
expect(tokenizedBuffer.firstInvalidRow()).toBe 5
buffer.setTextInRange([[6, 0], [7, 0]], "\n\n\n")
expect(tokenizedBuffer.lineForScreenRow(6).ruleStack?).toBeFalsy()
expect(tokenizedBuffer.lineForScreenRow(7).ruleStack?).toBeFalsy()
expect(tokenizedBuffer.tokenizedLineForRow(6).ruleStack?).toBeFalsy()
expect(tokenizedBuffer.tokenizedLineForRow(7).ruleStack?).toBeFalsy()
changeHandler.reset()
expect(tokenizedBuffer.firstInvalidRow()).toBe 5
@@ -149,10 +149,10 @@ describe "TokenizedBuffer", ->
it "updates tokens to reflect the change", ->
buffer.setTextInRange([[0, 0], [2, 0]], "foo()\n7\n")
expect(tokenizedBuffer.lineForScreenRow(0).tokens[1]).toEqual(value: '(', scopes: ['source.js', 'meta.brace.round.js'])
expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: '7', scopes: ['source.js', 'constant.numeric.js'])
expect(tokenizedBuffer.tokenizedLineForRow(0).tokens[1]).toEqual(value: '(', scopes: ['source.js', 'meta.brace.round.js'])
expect(tokenizedBuffer.tokenizedLineForRow(1).tokens[0]).toEqual(value: '7', scopes: ['source.js', 'constant.numeric.js'])
# line 2 is unchanged
expect(tokenizedBuffer.lineForScreenRow(2).tokens[2]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js'])
expect(tokenizedBuffer.tokenizedLineForRow(2).tokens[2]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js'])
expect(changeHandler).toHaveBeenCalled()
[event] = changeHandler.argsForCall[0]
@@ -164,7 +164,7 @@ describe "TokenizedBuffer", ->
buffer.insert([5, 30], '/* */')
changeHandler.reset()
buffer.insert([2, 0], '/*')
expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js']
expect(tokenizedBuffer.tokenizedLineForRow(3).tokens[0].scopes).toEqual ['source.js']
expect(changeHandler).toHaveBeenCalled()
[event] = changeHandler.argsForCall[0]
delete event.bufferChange
@@ -172,9 +172,9 @@ describe "TokenizedBuffer", ->
changeHandler.reset()
advanceClock()
expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.tokenizedLineForRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.tokenizedLineForRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(changeHandler).toHaveBeenCalled()
[event] = changeHandler.argsForCall[0]
delete event.bufferChange
@@ -185,23 +185,23 @@ describe "TokenizedBuffer", ->
buffer.insert([5, 0], '*/')
buffer.insert([1, 0], 'var ')
expect(tokenizedBuffer.lineForScreenRow(1).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.tokenizedLineForRow(1).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
describe "when lines are both updated and removed", ->
it "updates tokens to reflect the change", ->
buffer.setTextInRange([[1, 0], [3, 0]], "foo()")
# previous line 0 remains
expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.modifier.js'])
expect(tokenizedBuffer.tokenizedLineForRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.modifier.js'])
# previous line 3 should be combined with input to form line 1
expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js'])
expect(tokenizedBuffer.lineForScreenRow(1).tokens[6]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
expect(tokenizedBuffer.tokenizedLineForRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js'])
expect(tokenizedBuffer.tokenizedLineForRow(1).tokens[6]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
# lines below deleted regions should be shifted upward
expect(tokenizedBuffer.lineForScreenRow(2).tokens[2]).toEqual(value: 'while', scopes: ['source.js', 'keyword.control.js'])
expect(tokenizedBuffer.lineForScreenRow(3).tokens[4]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
expect(tokenizedBuffer.lineForScreenRow(4).tokens[4]).toEqual(value: '<', scopes: ['source.js', 'keyword.operator.js'])
expect(tokenizedBuffer.tokenizedLineForRow(2).tokens[2]).toEqual(value: 'while', scopes: ['source.js', 'keyword.control.js'])
expect(tokenizedBuffer.tokenizedLineForRow(3).tokens[4]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
expect(tokenizedBuffer.tokenizedLineForRow(4).tokens[4]).toEqual(value: '<', scopes: ['source.js', 'keyword.operator.js'])
expect(changeHandler).toHaveBeenCalled()
[event] = changeHandler.argsForCall[0]
@@ -214,8 +214,8 @@ describe "TokenizedBuffer", ->
changeHandler.reset()
buffer.setTextInRange([[2, 0], [3, 0]], '/*')
expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js']
expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js']
expect(tokenizedBuffer.tokenizedLineForRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js']
expect(tokenizedBuffer.tokenizedLineForRow(3).tokens[0].scopes).toEqual ['source.js']
expect(changeHandler).toHaveBeenCalled()
[event] = changeHandler.argsForCall[0]
delete event.bufferChange
@@ -223,8 +223,8 @@ describe "TokenizedBuffer", ->
changeHandler.reset()
advanceClock()
expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.tokenizedLineForRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.tokenizedLineForRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(changeHandler).toHaveBeenCalled()
[event] = changeHandler.argsForCall[0]
delete event.bufferChange
@@ -235,19 +235,19 @@ describe "TokenizedBuffer", ->
buffer.setTextInRange([[1, 0], [2, 0]], "foo()\nbar()\nbaz()\nquux()")
# previous line 0 remains
expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual( value: 'var', scopes: ['source.js', 'storage.modifier.js'])
expect(tokenizedBuffer.tokenizedLineForRow(0).tokens[0]).toEqual( value: 'var', scopes: ['source.js', 'storage.modifier.js'])
# 3 new lines inserted
expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js'])
expect(tokenizedBuffer.lineForScreenRow(2).tokens[0]).toEqual(value: 'bar', scopes: ['source.js'])
expect(tokenizedBuffer.lineForScreenRow(3).tokens[0]).toEqual(value: 'baz', scopes: ['source.js'])
expect(tokenizedBuffer.tokenizedLineForRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js'])
expect(tokenizedBuffer.tokenizedLineForRow(2).tokens[0]).toEqual(value: 'bar', scopes: ['source.js'])
expect(tokenizedBuffer.tokenizedLineForRow(3).tokens[0]).toEqual(value: 'baz', scopes: ['source.js'])
# previous line 2 is joined with quux() on line 4
expect(tokenizedBuffer.lineForScreenRow(4).tokens[0]).toEqual(value: 'quux', scopes: ['source.js'])
expect(tokenizedBuffer.lineForScreenRow(4).tokens[4]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js'])
expect(tokenizedBuffer.tokenizedLineForRow(4).tokens[0]).toEqual(value: 'quux', scopes: ['source.js'])
expect(tokenizedBuffer.tokenizedLineForRow(4).tokens[4]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js'])
# previous line 3 is pushed down to become line 5
expect(tokenizedBuffer.lineForScreenRow(5).tokens[4]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[4]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
expect(changeHandler).toHaveBeenCalled()
[event] = changeHandler.argsForCall[0]
@@ -264,17 +264,17 @@ describe "TokenizedBuffer", ->
[event] = changeHandler.argsForCall[0]
delete event.bufferChange
expect(event).toEqual(start: 2, end: 2, delta: 2)
expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js']
expect(tokenizedBuffer.lineForScreenRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.lineForScreenRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js']
expect(tokenizedBuffer.tokenizedLineForRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js']
expect(tokenizedBuffer.tokenizedLineForRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.tokenizedLineForRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[0].scopes).toEqual ['source.js']
changeHandler.reset()
advanceClock() # tokenize invalidated lines in background
expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.lineForScreenRow(6).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.lineForScreenRow(7).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.lineForScreenRow(8).tokens[0].scopes).not.toBe ['source.js', 'comment.block.js']
expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.tokenizedLineForRow(6).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.tokenizedLineForRow(7).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.tokenizedLineForRow(8).tokens[0].scopes).not.toBe ['source.js', 'comment.block.js']
expect(changeHandler).toHaveBeenCalled()
[event] = changeHandler.argsForCall[0]
@@ -285,13 +285,13 @@ describe "TokenizedBuffer", ->
it "tokenizes the initial chunk synchronously, then tokenizes the remaining lines in the background", ->
commentBlock = _.multiplyString("// a comment\n", tokenizedBuffer.chunkSize + 2)
buffer.insert([0,0], commentBlock)
expect(tokenizedBuffer.lineForScreenRow(0).ruleStack?).toBeTruthy()
expect(tokenizedBuffer.lineForScreenRow(4).ruleStack?).toBeTruthy()
expect(tokenizedBuffer.lineForScreenRow(5).ruleStack?).toBeFalsy()
expect(tokenizedBuffer.tokenizedLineForRow(0).ruleStack?).toBeTruthy()
expect(tokenizedBuffer.tokenizedLineForRow(4).ruleStack?).toBeTruthy()
expect(tokenizedBuffer.tokenizedLineForRow(5).ruleStack?).toBeFalsy()
advanceClock()
expect(tokenizedBuffer.lineForScreenRow(5).ruleStack?).toBeTruthy()
expect(tokenizedBuffer.lineForScreenRow(6).ruleStack?).toBeTruthy()
expect(tokenizedBuffer.tokenizedLineForRow(5).ruleStack?).toBeTruthy()
expect(tokenizedBuffer.tokenizedLineForRow(6).ruleStack?).toBeTruthy()
describe ".findOpeningBracket(closingBufferPosition)", ->
it "returns the position of the matching bracket, skipping any nested brackets", ->
@@ -302,18 +302,18 @@ describe "TokenizedBuffer", ->
expect(tokenizedBuffer.findClosingBracket([1, 29])).toEqual [9, 2]
it "tokenizes leading whitespace based on the new tab length", ->
expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].isAtomic).toBeTruthy()
expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].value).toBe " "
expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].isAtomic).toBeTruthy()
expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].value).toBe " "
expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[0].isAtomic).toBeTruthy()
expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[0].value).toBe " "
expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[1].isAtomic).toBeTruthy()
expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[1].value).toBe " "
tokenizedBuffer.setTabLength(4)
fullyTokenize(tokenizedBuffer)
expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].isAtomic).toBeTruthy()
expect(tokenizedBuffer.lineForScreenRow(5).tokens[0].value).toBe " "
expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].isAtomic).toBeFalsy()
expect(tokenizedBuffer.lineForScreenRow(5).tokens[1].value).toBe " current "
expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[0].isAtomic).toBeTruthy()
expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[0].value).toBe " "
expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[1].isAtomic).toBeFalsy()
expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[1].value).toBe " current "
describe "when the buffer contains hard-tabs", ->
beforeEach ->
@@ -335,7 +335,7 @@ describe "TokenizedBuffer", ->
it "renders each tab as its own atomic token with a value of size tabLength", ->
tabAsSpaces = _.multiplyString(' ', tokenizedBuffer.getTabLength())
screenLine0 = tokenizedBuffer.lineForScreenRow(0)
screenLine0 = tokenizedBuffer.tokenizedLineForRow(0)
expect(screenLine0.text).toBe "# Econ 101#{tabAsSpaces}"
{ tokens } = screenLine0
@@ -347,7 +347,7 @@ describe "TokenizedBuffer", ->
expect(tokens[2].isAtomic).toBeTruthy()
expect(tokens[3].value).toBe ""
expect(tokenizedBuffer.lineForScreenRow(2).text).toBe "#{tabAsSpaces} buy()#{tabAsSpaces}while supply > demand"
expect(tokenizedBuffer.tokenizedLineForRow(2).text).toBe "#{tabAsSpaces} buy()#{tabAsSpaces}while supply > demand"
it "aligns the hard tabs to the correct tab stop column", ->
buffer.setText """
@@ -359,62 +359,62 @@ describe "TokenizedBuffer", ->
tokenizedBuffer.setTabLength(4)
fullyTokenize(tokenizedBuffer)
expect(tokenizedBuffer.lineForScreenRow(0).text).toBe "1 2 3 4"
expect(tokenizedBuffer.lineForScreenRow(0).tokens[1].bufferDelta).toBe 1
expect(tokenizedBuffer.lineForScreenRow(0).tokens[1].screenDelta).toBe 3
expect(tokenizedBuffer.tokenizedLineForRow(0).text).toBe "1 2 3 4"
expect(tokenizedBuffer.tokenizedLineForRow(0).tokens[1].bufferDelta).toBe 1
expect(tokenizedBuffer.tokenizedLineForRow(0).tokens[1].screenDelta).toBe 3
expect(tokenizedBuffer.lineForScreenRow(1).text).toBe "12 3 4 5"
expect(tokenizedBuffer.lineForScreenRow(1).tokens[1].bufferDelta).toBe 1
expect(tokenizedBuffer.lineForScreenRow(1).tokens[1].screenDelta).toBe 2
expect(tokenizedBuffer.tokenizedLineForRow(1).text).toBe "12 3 4 5"
expect(tokenizedBuffer.tokenizedLineForRow(1).tokens[1].bufferDelta).toBe 1
expect(tokenizedBuffer.tokenizedLineForRow(1).tokens[1].screenDelta).toBe 2
expect(tokenizedBuffer.lineForScreenRow(2).text).toBe "123 4 5 6"
expect(tokenizedBuffer.lineForScreenRow(2).tokens[1].bufferDelta).toBe 1
expect(tokenizedBuffer.lineForScreenRow(2).tokens[1].screenDelta).toBe 1
expect(tokenizedBuffer.tokenizedLineForRow(2).text).toBe "123 4 5 6"
expect(tokenizedBuffer.tokenizedLineForRow(2).tokens[1].bufferDelta).toBe 1
expect(tokenizedBuffer.tokenizedLineForRow(2).tokens[1].screenDelta).toBe 1
tokenizedBuffer.setTabLength(3)
fullyTokenize(tokenizedBuffer)
expect(tokenizedBuffer.lineForScreenRow(0).text).toBe "1 2 3 4"
expect(tokenizedBuffer.lineForScreenRow(0).tokens[1].bufferDelta).toBe 1
expect(tokenizedBuffer.lineForScreenRow(0).tokens[1].screenDelta).toBe 2
expect(tokenizedBuffer.tokenizedLineForRow(0).text).toBe "1 2 3 4"
expect(tokenizedBuffer.tokenizedLineForRow(0).tokens[1].bufferDelta).toBe 1
expect(tokenizedBuffer.tokenizedLineForRow(0).tokens[1].screenDelta).toBe 2
expect(tokenizedBuffer.lineForScreenRow(1).text).toBe "12 3 4 5"
expect(tokenizedBuffer.lineForScreenRow(1).tokens[1].bufferDelta).toBe 1
expect(tokenizedBuffer.lineForScreenRow(1).tokens[1].screenDelta).toBe 1
expect(tokenizedBuffer.tokenizedLineForRow(1).text).toBe "12 3 4 5"
expect(tokenizedBuffer.tokenizedLineForRow(1).tokens[1].bufferDelta).toBe 1
expect(tokenizedBuffer.tokenizedLineForRow(1).tokens[1].screenDelta).toBe 1
expect(tokenizedBuffer.lineForScreenRow(2).text).toBe "123 4 5 6"
expect(tokenizedBuffer.lineForScreenRow(2).tokens[1].bufferDelta).toBe 1
expect(tokenizedBuffer.lineForScreenRow(2).tokens[1].screenDelta).toBe 3
expect(tokenizedBuffer.tokenizedLineForRow(2).text).toBe "123 4 5 6"
expect(tokenizedBuffer.tokenizedLineForRow(2).tokens[1].bufferDelta).toBe 1
expect(tokenizedBuffer.tokenizedLineForRow(2).tokens[1].screenDelta).toBe 3
tokenizedBuffer.setTabLength(2)
fullyTokenize(tokenizedBuffer)
expect(tokenizedBuffer.lineForScreenRow(0).text).toBe "1 2 3 4"
expect(tokenizedBuffer.lineForScreenRow(0).tokens[1].bufferDelta).toBe 1
expect(tokenizedBuffer.lineForScreenRow(0).tokens[1].screenDelta).toBe 1
expect(tokenizedBuffer.tokenizedLineForRow(0).text).toBe "1 2 3 4"
expect(tokenizedBuffer.tokenizedLineForRow(0).tokens[1].bufferDelta).toBe 1
expect(tokenizedBuffer.tokenizedLineForRow(0).tokens[1].screenDelta).toBe 1
expect(tokenizedBuffer.lineForScreenRow(1).text).toBe "12 3 4 5"
expect(tokenizedBuffer.lineForScreenRow(1).tokens[1].bufferDelta).toBe 1
expect(tokenizedBuffer.lineForScreenRow(1).tokens[1].screenDelta).toBe 2
expect(tokenizedBuffer.tokenizedLineForRow(1).text).toBe "12 3 4 5"
expect(tokenizedBuffer.tokenizedLineForRow(1).tokens[1].bufferDelta).toBe 1
expect(tokenizedBuffer.tokenizedLineForRow(1).tokens[1].screenDelta).toBe 2
expect(tokenizedBuffer.lineForScreenRow(2).text).toBe "123 4 5 6"
expect(tokenizedBuffer.lineForScreenRow(2).tokens[1].bufferDelta).toBe 1
expect(tokenizedBuffer.lineForScreenRow(2).tokens[1].screenDelta).toBe 1
expect(tokenizedBuffer.tokenizedLineForRow(2).text).toBe "123 4 5 6"
expect(tokenizedBuffer.tokenizedLineForRow(2).tokens[1].bufferDelta).toBe 1
expect(tokenizedBuffer.tokenizedLineForRow(2).tokens[1].screenDelta).toBe 1
tokenizedBuffer.setTabLength(1)
fullyTokenize(tokenizedBuffer)
expect(tokenizedBuffer.lineForScreenRow(0).text).toBe "1 2 3 4"
expect(tokenizedBuffer.lineForScreenRow(0).tokens[1].bufferDelta).toBe 1
expect(tokenizedBuffer.lineForScreenRow(0).tokens[1].screenDelta).toBe 1
expect(tokenizedBuffer.tokenizedLineForRow(0).text).toBe "1 2 3 4"
expect(tokenizedBuffer.tokenizedLineForRow(0).tokens[1].bufferDelta).toBe 1
expect(tokenizedBuffer.tokenizedLineForRow(0).tokens[1].screenDelta).toBe 1
expect(tokenizedBuffer.lineForScreenRow(1).text).toBe "12 3 4 5"
expect(tokenizedBuffer.lineForScreenRow(1).tokens[1].bufferDelta).toBe 1
expect(tokenizedBuffer.lineForScreenRow(1).tokens[1].screenDelta).toBe 1
expect(tokenizedBuffer.tokenizedLineForRow(1).text).toBe "12 3 4 5"
expect(tokenizedBuffer.tokenizedLineForRow(1).tokens[1].bufferDelta).toBe 1
expect(tokenizedBuffer.tokenizedLineForRow(1).tokens[1].screenDelta).toBe 1
expect(tokenizedBuffer.lineForScreenRow(2).text).toBe "123 4 5 6"
expect(tokenizedBuffer.lineForScreenRow(2).tokens[1].bufferDelta).toBe 1
expect(tokenizedBuffer.lineForScreenRow(2).tokens[1].screenDelta).toBe 1
expect(tokenizedBuffer.tokenizedLineForRow(2).text).toBe "123 4 5 6"
expect(tokenizedBuffer.tokenizedLineForRow(2).tokens[1].bufferDelta).toBe 1
expect(tokenizedBuffer.tokenizedLineForRow(2).tokens[1].screenDelta).toBe 1
describe "when the buffer contains UTF-8 surrogate pairs", ->
beforeEach ->
@@ -435,7 +435,7 @@ describe "TokenizedBuffer", ->
buffer.release()
it "renders each UTF-8 surrogate pair as its own atomic token", ->
screenLine0 = tokenizedBuffer.lineForScreenRow(0)
screenLine0 = tokenizedBuffer.tokenizedLineForRow(0)
expect(screenLine0.text).toBe "'abc\uD835\uDF97def'"
{ tokens } = screenLine0
@@ -447,7 +447,7 @@ describe "TokenizedBuffer", ->
expect(tokens[3].value).toBe "def"
expect(tokens[4].value).toBe "'"
screenLine1 = tokenizedBuffer.lineForScreenRow(1)
screenLine1 = tokenizedBuffer.tokenizedLineForRow(1)
expect(screenLine1.text).toBe "//\uD835\uDF97xyz"
{ tokens } = screenLine1
@@ -525,7 +525,7 @@ describe "TokenizedBuffer", ->
tokenizedBuffer.setGrammar(atom.syntax.selectGrammar('test.erb'))
fullyTokenize(tokenizedBuffer)
{tokens} = tokenizedBuffer.lineForScreenRow(0)
{tokens} = tokenizedBuffer.tokenizedLineForRow(0)
expect(tokens[0]).toEqual value: "<div class='name'>", scopes: ["text.html.ruby"]
waitsForPromise ->
@@ -533,7 +533,7 @@ describe "TokenizedBuffer", ->
runs ->
fullyTokenize(tokenizedBuffer)
{tokens} = tokenizedBuffer.lineForScreenRow(0)
{tokens} = tokenizedBuffer.tokenizedLineForRow(0)
expect(tokens[0]).toEqual value: '<', scopes: ["text.html.ruby","meta.tag.block.any.html","punctuation.definition.tag.begin.html"]
describe ".tokenForPosition(position)", ->
@@ -599,9 +599,9 @@ describe "TokenizedBuffer", ->
tokenizedBuffer.setInvisibles(space: 'S', tab: 'T')
fullyTokenize(tokenizedBuffer)
expect(tokenizedBuffer.lineForScreenRow(0).text).toBe "SST Sa line with tabsTand T spacesSTS"
expect(tokenizedBuffer.tokenizedLineForRow(0).text).toBe "SST Sa line with tabsTand T spacesSTS"
# Also needs to work for copies
expect(tokenizedBuffer.lineForScreenRow(0).copy().text).toBe "SST Sa line with tabsTand T spacesSTS"
expect(tokenizedBuffer.tokenizedLineForRow(0).copy().text).toBe "SST Sa line with tabsTand T spacesSTS"
it "assigns endOfLineInvisibles to tokenized lines", ->
buffer = new TextBuffer(text: "a line that ends in a carriage-return-line-feed \r\na line that ends in just a line-feed\na line with no ending")
@@ -611,17 +611,17 @@ describe "TokenizedBuffer", ->
tokenizedBuffer.setInvisibles(cr: 'R', eol: 'N')
fullyTokenize(tokenizedBuffer)
expect(tokenizedBuffer.lineForScreenRow(0).endOfLineInvisibles).toEqual ['R', 'N']
expect(tokenizedBuffer.lineForScreenRow(1).endOfLineInvisibles).toEqual ['N']
expect(tokenizedBuffer.tokenizedLineForRow(0).endOfLineInvisibles).toEqual ['R', 'N']
expect(tokenizedBuffer.tokenizedLineForRow(1).endOfLineInvisibles).toEqual ['N']
# Lines ending in soft wraps get no invisibles
[left, right] = tokenizedBuffer.lineForScreenRow(0).softWrapAt(20)
[left, right] = tokenizedBuffer.tokenizedLineForRow(0).softWrapAt(20)
expect(left.endOfLineInvisibles).toBe null
expect(right.endOfLineInvisibles).toEqual ['R', 'N']
tokenizedBuffer.setInvisibles(cr: 'R', eol: false)
expect(tokenizedBuffer.lineForScreenRow(0).endOfLineInvisibles).toEqual ['R']
expect(tokenizedBuffer.lineForScreenRow(1).endOfLineInvisibles).toEqual []
expect(tokenizedBuffer.tokenizedLineForRow(0).endOfLineInvisibles).toEqual ['R']
expect(tokenizedBuffer.tokenizedLineForRow(1).endOfLineInvisibles).toEqual []
describe "leading and trailing whitespace", ->
beforeEach ->
@@ -630,41 +630,40 @@ describe "TokenizedBuffer", ->
fullyTokenize(tokenizedBuffer)
it "assigns ::firstNonWhitespaceIndex on tokens that have leading whitespace", ->
expect(tokenizedBuffer.lineForScreenRow(0).tokens[0].firstNonWhitespaceIndex).toBe null
expect(tokenizedBuffer.lineForScreenRow(1).tokens[0].firstNonWhitespaceIndex).toBe 2
expect(tokenizedBuffer.lineForScreenRow(1).tokens[1].firstNonWhitespaceIndex).toBe null
expect(tokenizedBuffer.tokenizedLineForRow(0).tokens[0].firstNonWhitespaceIndex).toBe null
expect(tokenizedBuffer.tokenizedLineForRow(1).tokens[0].firstNonWhitespaceIndex).toBe 2
expect(tokenizedBuffer.tokenizedLineForRow(1).tokens[1].firstNonWhitespaceIndex).toBe null
expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].firstNonWhitespaceIndex).toBe 2
expect(tokenizedBuffer.lineForScreenRow(2).tokens[1].firstNonWhitespaceIndex).toBe 2
expect(tokenizedBuffer.lineForScreenRow(2).tokens[2].firstNonWhitespaceIndex).toBe null
expect(tokenizedBuffer.tokenizedLineForRow(2).tokens[0].firstNonWhitespaceIndex).toBe 2
expect(tokenizedBuffer.tokenizedLineForRow(2).tokens[1].firstNonWhitespaceIndex).toBe 2
expect(tokenizedBuffer.tokenizedLineForRow(2).tokens[2].firstNonWhitespaceIndex).toBe null
# The 4th token *has* leading whitespace, but isn't entirely whitespace
buffer.insert([5, 0], ' ')
expect(tokenizedBuffer.lineForScreenRow(5).tokens[3].firstNonWhitespaceIndex).toBe 1
expect(tokenizedBuffer.lineForScreenRow(5).tokens[4].firstNonWhitespaceIndex).toBe null
expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[3].firstNonWhitespaceIndex).toBe 1
expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[4].firstNonWhitespaceIndex).toBe null
# Lines that are *only* whitespace are not considered to have leading whitespace
buffer.insert([10, 0], ' ')
expect(tokenizedBuffer.lineForScreenRow(10).tokens[0].firstNonWhitespaceIndex).toBe null
expect(tokenizedBuffer.tokenizedLineForRow(10).tokens[0].firstNonWhitespaceIndex).toBe null
it "assigns ::firstTrailingWhitespaceIndex on tokens that have trailing whitespace", ->
buffer.insert([0, Infinity], ' ')
expect(tokenizedBuffer.lineForScreenRow(0).tokens[11].firstTrailingWhitespaceIndex).toBe null
expect(tokenizedBuffer.lineForScreenRow(0).tokens[12].firstTrailingWhitespaceIndex).toBe 0
expect(tokenizedBuffer.tokenizedLineForRow(0).tokens[11].firstTrailingWhitespaceIndex).toBe null
expect(tokenizedBuffer.tokenizedLineForRow(0).tokens[12].firstTrailingWhitespaceIndex).toBe 0
# The last token *has* trailing whitespace, but isn't entirely whitespace
buffer.setTextInRange([[2, 39], [2, 40]], ' ')
expect(tokenizedBuffer.lineForScreenRow(2).tokens[14].firstTrailingWhitespaceIndex).toBe null
console.log tokenizedBuffer.lineForScreenRow(2).tokens[15]
expect(tokenizedBuffer.lineForScreenRow(2).tokens[15].firstTrailingWhitespaceIndex).toBe 6
expect(tokenizedBuffer.tokenizedLineForRow(2).tokens[14].firstTrailingWhitespaceIndex).toBe null
expect(tokenizedBuffer.tokenizedLineForRow(2).tokens[15].firstTrailingWhitespaceIndex).toBe 6
# Lines that are *only* whitespace are considered to have trailing whitespace
buffer.insert([10, 0], ' ')
expect(tokenizedBuffer.lineForScreenRow(10).tokens[0].firstTrailingWhitespaceIndex).toBe 0
expect(tokenizedBuffer.tokenizedLineForRow(10).tokens[0].firstTrailingWhitespaceIndex).toBe 0
it "only marks trailing whitespace on the last segment of a soft-wrapped line", ->
buffer.insert([0, Infinity], ' ')
tokenizedLine = tokenizedBuffer.lineForScreenRow(0)
tokenizedLine = tokenizedBuffer.tokenizedLineForRow(0)
[segment1, segment2] = tokenizedLine.softWrapAt(16)
expect(segment1.tokens[5].value).toBe ' '
expect(segment1.tokens[5].firstTrailingWhitespaceIndex).toBe null
@@ -677,7 +676,7 @@ describe "TokenizedBuffer", ->
tokenizedBuffer.setInvisibles(space: 'S', tab: 'T')
fullyTokenize(tokenizedBuffer)
line = tokenizedBuffer.lineForScreenRow(0).copy()
line = tokenizedBuffer.tokenizedLineForRow(0).copy()
expect(line.tokens[0].firstNonWhitespaceIndex).toBe 2
expect(line.tokens[line.tokens.length - 1].firstTrailingWhitespaceIndex).toBe 0
@@ -704,55 +703,55 @@ describe "TokenizedBuffer", ->
describe "when the line is non-empty", ->
it "has an indent level based on the leading whitespace on the line", ->
expect(tokenizedBuffer.lineForScreenRow(0).indentLevel).toBe 0
expect(tokenizedBuffer.lineForScreenRow(1).indentLevel).toBe 1
expect(tokenizedBuffer.lineForScreenRow(2).indentLevel).toBe 2
expect(tokenizedBuffer.tokenizedLineForRow(0).indentLevel).toBe 0
expect(tokenizedBuffer.tokenizedLineForRow(1).indentLevel).toBe 1
expect(tokenizedBuffer.tokenizedLineForRow(2).indentLevel).toBe 2
buffer.insert([2, 0], ' ')
expect(tokenizedBuffer.lineForScreenRow(2).indentLevel).toBe 2.5
expect(tokenizedBuffer.tokenizedLineForRow(2).indentLevel).toBe 2.5
describe "when the line is empty", ->
it "assumes the indentation level of the first non-empty line below or above if one exists", ->
buffer.insert([12, 0], ' ')
buffer.insert([12, Infinity], '\n\n')
expect(tokenizedBuffer.lineForScreenRow(13).indentLevel).toBe 2
expect(tokenizedBuffer.lineForScreenRow(14).indentLevel).toBe 2
expect(tokenizedBuffer.tokenizedLineForRow(13).indentLevel).toBe 2
expect(tokenizedBuffer.tokenizedLineForRow(14).indentLevel).toBe 2
buffer.insert([1, Infinity], '\n\n')
expect(tokenizedBuffer.lineForScreenRow(2).indentLevel).toBe 2
expect(tokenizedBuffer.lineForScreenRow(3).indentLevel).toBe 2
expect(tokenizedBuffer.tokenizedLineForRow(2).indentLevel).toBe 2
expect(tokenizedBuffer.tokenizedLineForRow(3).indentLevel).toBe 2
buffer.setText('\n\n\n')
expect(tokenizedBuffer.lineForScreenRow(1).indentLevel).toBe 0
expect(tokenizedBuffer.tokenizedLineForRow(1).indentLevel).toBe 0
describe "when the changed lines are surrounded by whitespace-only lines", ->
it "updates the indentLevel of empty lines that precede the change", ->
expect(tokenizedBuffer.lineForScreenRow(12).indentLevel).toBe 0
expect(tokenizedBuffer.tokenizedLineForRow(12).indentLevel).toBe 0
buffer.insert([12, 0], '\n')
buffer.insert([13, 0], ' ')
expect(tokenizedBuffer.lineForScreenRow(12).indentLevel).toBe 1
expect(tokenizedBuffer.tokenizedLineForRow(12).indentLevel).toBe 1
it "updates empty line indent guides when the empty line is the last line", ->
buffer.insert([12, 2], '\n')
# The newline and he tab need to be in two different operations to surface the bug
buffer.insert([12, 0], ' ')
expect(tokenizedBuffer.lineForScreenRow(13).indentLevel).toBe 1
expect(tokenizedBuffer.tokenizedLineForRow(13).indentLevel).toBe 1
buffer.insert([12, 0], ' ')
expect(tokenizedBuffer.lineForScreenRow(13).indentLevel).toBe 2
expect(tokenizedBuffer.lineForScreenRow(14)).not.toBeDefined()
expect(tokenizedBuffer.tokenizedLineForRow(13).indentLevel).toBe 2
expect(tokenizedBuffer.tokenizedLineForRow(14)).not.toBeDefined()
it "updates the indentLevel of empty lines surrounding a change that inserts lines", ->
# create some new lines
buffer.insert([7, 0], '\n\n')
buffer.insert([5, 0], '\n\n')
expect(tokenizedBuffer.lineForScreenRow(5).indentLevel).toBe 3
expect(tokenizedBuffer.lineForScreenRow(6).indentLevel).toBe 3
expect(tokenizedBuffer.lineForScreenRow(9).indentLevel).toBe 3
expect(tokenizedBuffer.lineForScreenRow(10).indentLevel).toBe 3
expect(tokenizedBuffer.lineForScreenRow(11).indentLevel).toBe 2
expect(tokenizedBuffer.tokenizedLineForRow(5).indentLevel).toBe 3
expect(tokenizedBuffer.tokenizedLineForRow(6).indentLevel).toBe 3
expect(tokenizedBuffer.tokenizedLineForRow(9).indentLevel).toBe 3
expect(tokenizedBuffer.tokenizedLineForRow(10).indentLevel).toBe 3
expect(tokenizedBuffer.tokenizedLineForRow(11).indentLevel).toBe 2
tokenizedBuffer.on "changed", changeHandler = jasmine.createSpy('changeHandler')
@@ -761,11 +760,11 @@ describe "TokenizedBuffer", ->
delete changeHandler.argsForCall[0][0].bufferChange
expect(changeHandler).toHaveBeenCalledWith(start: 5, end: 10, delta: 2)
expect(tokenizedBuffer.lineForScreenRow(5).indentLevel).toBe 4
expect(tokenizedBuffer.lineForScreenRow(6).indentLevel).toBe 4
expect(tokenizedBuffer.lineForScreenRow(11).indentLevel).toBe 4
expect(tokenizedBuffer.lineForScreenRow(12).indentLevel).toBe 4
expect(tokenizedBuffer.lineForScreenRow(13).indentLevel).toBe 2
expect(tokenizedBuffer.tokenizedLineForRow(5).indentLevel).toBe 4
expect(tokenizedBuffer.tokenizedLineForRow(6).indentLevel).toBe 4
expect(tokenizedBuffer.tokenizedLineForRow(11).indentLevel).toBe 4
expect(tokenizedBuffer.tokenizedLineForRow(12).indentLevel).toBe 4
expect(tokenizedBuffer.tokenizedLineForRow(13).indentLevel).toBe 2
it "updates the indentLevel of empty lines surrounding a change that removes lines", ->
# create some new lines
@@ -779,9 +778,9 @@ describe "TokenizedBuffer", ->
delete changeHandler.argsForCall[0][0].bufferChange
expect(changeHandler).toHaveBeenCalledWith(start: 5, end: 10, delta: -1)
expect(tokenizedBuffer.lineForScreenRow(5).indentLevel).toBe 2
expect(tokenizedBuffer.lineForScreenRow(6).indentLevel).toBe 2
expect(tokenizedBuffer.lineForScreenRow(7).indentLevel).toBe 2 # new text
expect(tokenizedBuffer.lineForScreenRow(8).indentLevel).toBe 2
expect(tokenizedBuffer.lineForScreenRow(9).indentLevel).toBe 2
expect(tokenizedBuffer.lineForScreenRow(10).indentLevel).toBe 2 # }
expect(tokenizedBuffer.tokenizedLineForRow(5).indentLevel).toBe 2
expect(tokenizedBuffer.tokenizedLineForRow(6).indentLevel).toBe 2
expect(tokenizedBuffer.tokenizedLineForRow(7).indentLevel).toBe 2 # new text
expect(tokenizedBuffer.tokenizedLineForRow(8).indentLevel).toBe 2
expect(tokenizedBuffer.tokenizedLineForRow(9).indentLevel).toBe 2
expect(tokenizedBuffer.tokenizedLineForRow(10).indentLevel).toBe 2 # }

View File

@@ -1024,7 +1024,7 @@ class DisplayBuffer extends Model
bufferRow = startBufferRow
while bufferRow < endBufferRow
tokenizedLine = @tokenizedBuffer.lineForScreenRow(bufferRow)
tokenizedLine = @tokenizedBuffer.tokenizedLineForRow(bufferRow)
if fold = @largestFoldStartingAtBufferRow(bufferRow)
foldLine = tokenizedLine.copy()

View File

@@ -837,7 +837,7 @@ class Editor extends Model
# whitespace.
usesSoftTabs: ->
for bufferRow in [0..@buffer.getLastRow()]
continue if @displayBuffer.tokenizedBuffer.lineForScreenRow(bufferRow).isComment()
continue if @displayBuffer.tokenizedBuffer.tokenizedLineForRow(bufferRow).isComment()
line = @buffer.lineForRow(bufferRow)
return true if line[0] is ' '

View File

@@ -145,17 +145,17 @@ class LanguageMode
rowRange
rowRangeForCommentAtBufferRow: (bufferRow) ->
return unless @editor.displayBuffer.tokenizedBuffer.lineForScreenRow(bufferRow).isComment()
return unless @editor.displayBuffer.tokenizedBuffer.tokenizedLineForRow(bufferRow).isComment()
startRow = bufferRow
for currentRow in [bufferRow-1..0]
break if @buffer.isRowBlank(currentRow)
break unless @editor.displayBuffer.tokenizedBuffer.lineForScreenRow(currentRow).isComment()
break unless @editor.displayBuffer.tokenizedBuffer.tokenizedLineForRow(currentRow).isComment()
startRow = currentRow
endRow = bufferRow
for currentRow in [bufferRow+1..@buffer.getLastRow()]
break if @buffer.isRowBlank(currentRow)
break unless @editor.displayBuffer.tokenizedBuffer.lineForScreenRow(currentRow).isComment()
break unless @editor.displayBuffer.tokenizedBuffer.tokenizedLineForRow(currentRow).isComment()
endRow = currentRow
return [startRow, endRow] if startRow isnt endRow
@@ -201,7 +201,7 @@ class LanguageMode
# row is a comment.
isLineCommentedAtBufferRow: (bufferRow) ->
return false unless 0 <= bufferRow <= @editor.getLastBufferRow()
@editor.displayBuffer.tokenizedBuffer.lineForScreenRow(bufferRow).isComment()
@editor.displayBuffer.tokenizedBuffer.tokenizedLineForRow(bufferRow).isComment()
# Find a row range for a 'paragraph' around specified bufferRow.
# Right now, a paragraph is a block of text bounded by and empty line or a

View File

@@ -221,10 +221,8 @@ class TokenizedBuffer extends Model
{tokens, ruleStack} = @grammar.tokenizeLine(line, ruleStack, row is 0)
new TokenizedLine({tokens, ruleStack, tabLength, lineEnding, indentLevel, @invisibles})
# FIXME: benogle says: These are actually buffer rows as all buffer rows are
# accounted for in @tokenizedLines
lineForScreenRow: (row) ->
@linesForScreenRows(row, row)[0]
tokenizedLineForRow: (bufferRow) ->
@tokenizedLines[bufferRow]
# FIXME: benogle says: These are actually buffer rows as all buffer rows are
# accounted for in @tokenizedLines