diff --git a/spec/text-editor-registry-spec.js b/spec/text-editor-registry-spec.js index 86bb71a6f..51027e63c 100644 --- a/spec/text-editor-registry-spec.js +++ b/spec/text-editor-registry-spec.js @@ -198,13 +198,13 @@ describe('TextEditorRegistry', function () { registry.maintainConfig(editor2) await initialPackageActivation - expect(editor.getRootScopeDescriptor().getScopesArray()).toEqual(['text.plain']) + expect(editor.getRootScopeDescriptor().getScopesArray()).toEqual(['text.plain.null-grammar']) expect(editor2.getRootScopeDescriptor().getScopesArray()).toEqual(['source.js']) expect(editor.getEncoding()).toBe('utf8') expect(editor2.getEncoding()).toBe('utf8') - atom.config.set('core.fileEncoding', 'utf16le', {scopeSelector: '.text.plain'}) + atom.config.set('core.fileEncoding', 'utf16le', {scopeSelector: '.text.plain.null-grammar'}) atom.config.set('core.fileEncoding', 'utf16be', {scopeSelector: '.source.js'}) expect(editor.getEncoding()).toBe('utf16le') diff --git a/spec/token-iterator-spec.coffee b/spec/token-iterator-spec.coffee index f876d30d1..6ae01cd30 100644 --- a/spec/token-iterator-spec.coffee +++ b/spec/token-iterator-spec.coffee @@ -29,7 +29,7 @@ describe "TokenIterator", -> }) tokenizedBuffer.setGrammar(grammar) - tokenIterator = tokenizedBuffer.tokenizedLineForRow(1).getTokenIterator() + tokenIterator = tokenizedBuffer.tokenizedLines[1].getTokenIterator() tokenIterator.next() expect(tokenIterator.getBufferStart()).toBe 0 diff --git a/spec/tokenized-buffer-spec.coffee b/spec/tokenized-buffer-spec.coffee index ad9fa0ee7..6558d42b4 100644 --- a/spec/tokenized-buffer-spec.coffee +++ b/spec/tokenized-buffer-spec.coffee @@ -1,3 +1,4 @@ +NullGrammar = require '../src/null-grammar' TokenizedBuffer = require '../src/tokenized-buffer' {Point} = TextBuffer = require 'text-buffer' _ = require 'underscore-plus' @@ -32,15 +33,8 @@ describe "TokenizedBuffer", -> atom.packages.activatePackage('language-coffee-script') it "deserializes it searching among the buffers in the current project", -> - tokenizedBufferA = new TokenizedBuffer({ - buffer, grammarRegistry: atom.grammars, packageManager: atom.packages, - assert: atom.assert, tabLength: 2, - }) - tokenizedBufferB = TokenizedBuffer.deserialize( - JSON.parse(JSON.stringify(tokenizedBufferA.serialize())), - atom - ) - + tokenizedBufferA = new TokenizedBuffer({buffer, tabLength: 2}) + tokenizedBufferB = TokenizedBuffer.deserialize(JSON.parse(JSON.stringify(tokenizedBufferA.serialize())), atom) expect(tokenizedBufferB.buffer).toBe(tokenizedBufferA.buffer) describe "when the underlying buffer has no path", -> @@ -48,25 +42,14 @@ describe "TokenizedBuffer", -> buffer = atom.project.bufferForPathSync(null) it "deserializes it searching among the buffers in the current project", -> - tokenizedBufferA = new TokenizedBuffer({ - buffer, grammarRegistry: atom.grammars, packageManager: atom.packages, - assert: atom.assert, tabLength: 2, - }) - tokenizedBufferB = TokenizedBuffer.deserialize( - JSON.parse(JSON.stringify(tokenizedBufferA.serialize())), - atom - ) - + tokenizedBufferA = new TokenizedBuffer({buffer, tabLength: 2}) + tokenizedBufferB = TokenizedBuffer.deserialize(JSON.parse(JSON.stringify(tokenizedBufferA.serialize())), atom) expect(tokenizedBufferB.buffer).toBe(tokenizedBufferA.buffer) describe "when the buffer is destroyed", -> beforeEach -> buffer = atom.project.bufferForPathSync('sample.js') - tokenizedBuffer = new TokenizedBuffer({ - buffer, grammarRegistry: atom.grammars, packageManager: atom.packages, - assert: atom.assert, tabLength: 2, - }) - tokenizedBuffer.setGrammar(atom.grammars.grammarForScopeName('source.js')) + tokenizedBuffer = new TokenizedBuffer({buffer, grammar: atom.grammars.grammarForScopeName('source.js'), tabLength: 2}) startTokenizing(tokenizedBuffer) it "stops tokenization", -> @@ -78,11 +61,7 @@ describe "TokenizedBuffer", -> describe "when the buffer contains soft-tabs", -> beforeEach -> buffer = atom.project.bufferForPathSync('sample.js') - tokenizedBuffer = new TokenizedBuffer({ - buffer, grammarRegistry: atom.grammars, packageManager: atom.packages, - assert: atom.assert, tabLength: 2, - }) - tokenizedBuffer.setGrammar(atom.grammars.grammarForScopeName('source.js')) + tokenizedBuffer = new TokenizedBuffer({buffer, grammar: atom.grammars.grammarForScopeName('source.js'), tabLength: 2}) startTokenizing(tokenizedBuffer) afterEach -> @@ -90,32 +69,29 @@ describe "TokenizedBuffer", -> buffer.release() describe "on construction", -> - it "initially creates un-tokenized screen lines, then tokenizes lines chunk at a time in the background", -> - line0 = tokenizedBuffer.tokenizedLineForRow(0) - expect(line0.tokens).toEqual([value: line0.text, scopes: ['source.js']]) + it "tokenizes lines chunk at a time in the background", -> + line0 = tokenizedBuffer.tokenizedLines[0] + expect(line0).toBeUndefined() - line11 = tokenizedBuffer.tokenizedLineForRow(11) - expect(line11.tokens).toEqual([value: " return sort(Array.apply(this, arguments));", scopes: ['source.js']]) - - # background tokenization has not begun - expect(tokenizedBuffer.tokenizedLineForRow(0).ruleStack).toBeUndefined() + line11 = tokenizedBuffer.tokenizedLines[11] + expect(line11).toBeUndefined() # tokenize chunk 1 advanceClock() - expect(tokenizedBuffer.tokenizedLineForRow(0).ruleStack?).toBeTruthy() - expect(tokenizedBuffer.tokenizedLineForRow(4).ruleStack?).toBeTruthy() - expect(tokenizedBuffer.tokenizedLineForRow(5).ruleStack?).toBeFalsy() + expect(tokenizedBuffer.tokenizedLines[0].ruleStack?).toBeTruthy() + expect(tokenizedBuffer.tokenizedLines[4].ruleStack?).toBeTruthy() + expect(tokenizedBuffer.tokenizedLines[5]).toBeUndefined() # tokenize chunk 2 advanceClock() - expect(tokenizedBuffer.tokenizedLineForRow(5).ruleStack?).toBeTruthy() - expect(tokenizedBuffer.tokenizedLineForRow(9).ruleStack?).toBeTruthy() - expect(tokenizedBuffer.tokenizedLineForRow(10).ruleStack?).toBeFalsy() + expect(tokenizedBuffer.tokenizedLines[5].ruleStack?).toBeTruthy() + expect(tokenizedBuffer.tokenizedLines[9].ruleStack?).toBeTruthy() + expect(tokenizedBuffer.tokenizedLines[10]).toBeUndefined() # tokenize last chunk advanceClock() - expect(tokenizedBuffer.tokenizedLineForRow(10).ruleStack?).toBeTruthy() - expect(tokenizedBuffer.tokenizedLineForRow(12).ruleStack?).toBeTruthy() + expect(tokenizedBuffer.tokenizedLines[10].ruleStack?).toBeTruthy() + expect(tokenizedBuffer.tokenizedLines[12].ruleStack?).toBeTruthy() describe "when the buffer is partially tokenized", -> beforeEach -> @@ -152,8 +128,8 @@ describe "TokenizedBuffer", -> it "does not attempt to tokenize the lines in the change, and preserves the existing invalid row", -> expect(tokenizedBuffer.firstInvalidRow()).toBe 5 buffer.setTextInRange([[6, 0], [7, 0]], "\n\n\n") - expect(tokenizedBuffer.tokenizedLineForRow(6).ruleStack?).toBeFalsy() - expect(tokenizedBuffer.tokenizedLineForRow(7).ruleStack?).toBeFalsy() + expect(tokenizedBuffer.tokenizedLines[6]).toBeUndefined() + expect(tokenizedBuffer.tokenizedLines[7]).toBeUndefined() expect(tokenizedBuffer.firstInvalidRow()).toBe 5 describe "when the buffer is fully tokenized", -> @@ -165,101 +141,101 @@ describe "TokenizedBuffer", -> it "updates tokens to reflect the change", -> buffer.setTextInRange([[0, 0], [2, 0]], "foo()\n7\n") - expect(tokenizedBuffer.tokenizedLineForRow(0).tokens[1]).toEqual(value: '(', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js']) - expect(tokenizedBuffer.tokenizedLineForRow(1).tokens[0]).toEqual(value: '7', scopes: ['source.js', 'constant.numeric.decimal.js']) + expect(tokenizedBuffer.tokenizedLines[0].tokens[1]).toEqual(value: '(', scopes: ['source.js', 'meta.function-call.js', 'meta.arguments.js', 'punctuation.definition.arguments.begin.bracket.round.js']) + expect(tokenizedBuffer.tokenizedLines[1].tokens[0]).toEqual(value: '7', scopes: ['source.js', 'constant.numeric.decimal.js']) # line 2 is unchanged - expect(tokenizedBuffer.tokenizedLineForRow(2).tokens[1]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js']) + expect(tokenizedBuffer.tokenizedLines[2].tokens[1]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js']) describe "when the change invalidates the tokenization of subsequent lines", -> it "schedules the invalidated lines to be tokenized in the background", -> buffer.insert([5, 30], '/* */') buffer.insert([2, 0], '/*') - expect(tokenizedBuffer.tokenizedLineForRow(3).tokens[0].scopes).toEqual ['source.js'] + expect(tokenizedBuffer.tokenizedLines[3].tokens[0].scopes).toEqual ['source.js'] advanceClock() - expect(tokenizedBuffer.tokenizedLineForRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - expect(tokenizedBuffer.tokenizedLineForRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.tokenizedLines[3].tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.tokenizedLines[4].tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.tokenizedLines[5].tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] it "resumes highlighting with the state of the previous line", -> buffer.insert([0, 0], '/*') buffer.insert([5, 0], '*/') buffer.insert([1, 0], 'var ') - expect(tokenizedBuffer.tokenizedLineForRow(1).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.tokenizedLines[1].tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] describe "when lines are both updated and removed", -> it "updates tokens to reflect the change", -> buffer.setTextInRange([[1, 0], [3, 0]], "foo()") # previous line 0 remains - expect(tokenizedBuffer.tokenizedLineForRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.type.var.js']) + expect(tokenizedBuffer.tokenizedLines[0].tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.type.var.js']) # previous line 3 should be combined with input to form line 1 - expect(tokenizedBuffer.tokenizedLineForRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js']) - expect(tokenizedBuffer.tokenizedLineForRow(1).tokens[6]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.assignment.js']) + expect(tokenizedBuffer.tokenizedLines[1].tokens[0]).toEqual(value: 'foo', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js']) + expect(tokenizedBuffer.tokenizedLines[1].tokens[6]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.assignment.js']) # lines below deleted regions should be shifted upward - expect(tokenizedBuffer.tokenizedLineForRow(2).tokens[1]).toEqual(value: 'while', scopes: ['source.js', 'keyword.control.js']) - expect(tokenizedBuffer.tokenizedLineForRow(3).tokens[1]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.assignment.js']) - expect(tokenizedBuffer.tokenizedLineForRow(4).tokens[1]).toEqual(value: '<', scopes: ['source.js', 'keyword.operator.comparison.js']) + expect(tokenizedBuffer.tokenizedLines[2].tokens[1]).toEqual(value: 'while', scopes: ['source.js', 'keyword.control.js']) + expect(tokenizedBuffer.tokenizedLines[3].tokens[1]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.assignment.js']) + expect(tokenizedBuffer.tokenizedLines[4].tokens[1]).toEqual(value: '<', scopes: ['source.js', 'keyword.operator.comparison.js']) describe "when the change invalidates the tokenization of subsequent lines", -> it "schedules the invalidated lines to be tokenized in the background", -> buffer.insert([5, 30], '/* */') buffer.setTextInRange([[2, 0], [3, 0]], '/*') - expect(tokenizedBuffer.tokenizedLineForRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] - expect(tokenizedBuffer.tokenizedLineForRow(3).tokens[0].scopes).toEqual ['source.js'] + expect(tokenizedBuffer.tokenizedLines[2].tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] + expect(tokenizedBuffer.tokenizedLines[3].tokens[0].scopes).toEqual ['source.js'] advanceClock() - expect(tokenizedBuffer.tokenizedLineForRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - expect(tokenizedBuffer.tokenizedLineForRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.tokenizedLines[3].tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.tokenizedLines[4].tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] describe "when lines are both updated and inserted", -> it "updates tokens to reflect the change", -> buffer.setTextInRange([[1, 0], [2, 0]], "foo()\nbar()\nbaz()\nquux()") # previous line 0 remains - expect(tokenizedBuffer.tokenizedLineForRow(0).tokens[0]).toEqual( value: 'var', scopes: ['source.js', 'storage.type.var.js']) + expect(tokenizedBuffer.tokenizedLines[0].tokens[0]).toEqual( value: 'var', scopes: ['source.js', 'storage.type.var.js']) # 3 new lines inserted - expect(tokenizedBuffer.tokenizedLineForRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js']) - expect(tokenizedBuffer.tokenizedLineForRow(2).tokens[0]).toEqual(value: 'bar', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js']) - expect(tokenizedBuffer.tokenizedLineForRow(3).tokens[0]).toEqual(value: 'baz', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js']) + expect(tokenizedBuffer.tokenizedLines[1].tokens[0]).toEqual(value: 'foo', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js']) + expect(tokenizedBuffer.tokenizedLines[2].tokens[0]).toEqual(value: 'bar', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js']) + expect(tokenizedBuffer.tokenizedLines[3].tokens[0]).toEqual(value: 'baz', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js']) # previous line 2 is joined with quux() on line 4 - expect(tokenizedBuffer.tokenizedLineForRow(4).tokens[0]).toEqual(value: 'quux', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js']) - expect(tokenizedBuffer.tokenizedLineForRow(4).tokens[4]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js']) + expect(tokenizedBuffer.tokenizedLines[4].tokens[0]).toEqual(value: 'quux', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js']) + expect(tokenizedBuffer.tokenizedLines[4].tokens[4]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js']) # previous line 3 is pushed down to become line 5 - expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[3]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.assignment.js']) + expect(tokenizedBuffer.tokenizedLines[5].tokens[3]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.assignment.js']) describe "when the change invalidates the tokenization of subsequent lines", -> it "schedules the invalidated lines to be tokenized in the background", -> buffer.insert([5, 30], '/* */') buffer.insert([2, 0], '/*\nabcde\nabcder') - expect(tokenizedBuffer.tokenizedLineForRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] - expect(tokenizedBuffer.tokenizedLineForRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - expect(tokenizedBuffer.tokenizedLineForRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[0].scopes).toEqual ['source.js'] + expect(tokenizedBuffer.tokenizedLines[2].tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js'] + expect(tokenizedBuffer.tokenizedLines[3].tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.tokenizedLines[4].tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.tokenizedLines[5].tokens[0].scopes).toEqual ['source.js'] advanceClock() # tokenize invalidated lines in background - expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - expect(tokenizedBuffer.tokenizedLineForRow(6).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - expect(tokenizedBuffer.tokenizedLineForRow(7).tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] - expect(tokenizedBuffer.tokenizedLineForRow(8).tokens[0].scopes).not.toBe ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.tokenizedLines[5].tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.tokenizedLines[6].tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.tokenizedLines[7].tokens[0].scopes).toEqual ['source.js', 'comment.block.js'] + expect(tokenizedBuffer.tokenizedLines[8].tokens[0].scopes).not.toBe ['source.js', 'comment.block.js'] describe "when there is an insertion that is larger than the chunk size", -> it "tokenizes the initial chunk synchronously, then tokenizes the remaining lines in the background", -> commentBlock = _.multiplyString("// a comment\n", tokenizedBuffer.chunkSize + 2) buffer.insert([0, 0], commentBlock) - expect(tokenizedBuffer.tokenizedLineForRow(0).ruleStack?).toBeTruthy() - expect(tokenizedBuffer.tokenizedLineForRow(4).ruleStack?).toBeTruthy() - expect(tokenizedBuffer.tokenizedLineForRow(5).ruleStack?).toBeFalsy() + expect(tokenizedBuffer.tokenizedLines[0].ruleStack?).toBeTruthy() + expect(tokenizedBuffer.tokenizedLines[4].ruleStack?).toBeTruthy() + expect(tokenizedBuffer.tokenizedLines[5]).toBeUndefined() advanceClock() - expect(tokenizedBuffer.tokenizedLineForRow(5).ruleStack?).toBeTruthy() - expect(tokenizedBuffer.tokenizedLineForRow(6).ruleStack?).toBeTruthy() + expect(tokenizedBuffer.tokenizedLines[5].ruleStack?).toBeTruthy() + expect(tokenizedBuffer.tokenizedLines[6].ruleStack?).toBeTruthy() it "does not break out soft tabs across a scope boundary", -> waitsForPromise -> @@ -284,11 +260,7 @@ describe "TokenizedBuffer", -> runs -> buffer = atom.project.bufferForPathSync('sample-with-tabs.coffee') - tokenizedBuffer = new TokenizedBuffer({ - buffer, grammarRegistry: atom.grammars, packageManager: atom.packages, - assert: atom.assert, tabLength: 2, - }) - tokenizedBuffer.setGrammar(atom.grammars.grammarForScopeName('source.coffee')) + tokenizedBuffer = new TokenizedBuffer({buffer, grammar: atom.grammars.grammarForScopeName('source.coffee'), tabLength: 2}) startTokenizing(tokenizedBuffer) afterEach -> @@ -352,7 +324,6 @@ describe "TokenizedBuffer", -> expect(tokenizedHandler.callCount).toBe(1) it "retokenizes the buffer", -> - waitsForPromise -> atom.packages.activatePackage('language-ruby-on-rails') @@ -362,14 +333,10 @@ describe "TokenizedBuffer", -> runs -> buffer = atom.project.bufferForPathSync() buffer.setText "
<%= User.find(2).full_name %>
" - tokenizedBuffer = new TokenizedBuffer({ - buffer, grammarRegistry: atom.grammars, packageManager: atom.packages, - assert: atom.assert, tabLength: 2, - }) - tokenizedBuffer.setGrammar(atom.grammars.selectGrammar('test.erb')) + tokenizedBuffer = new TokenizedBuffer({buffer, grammar: atom.grammars.selectGrammar('test.erb'), tabLength: 2}) fullyTokenize(tokenizedBuffer) - {tokens} = tokenizedBuffer.tokenizedLineForRow(0) + {tokens} = tokenizedBuffer.tokenizedLines[0] expect(tokens[0]).toEqual value: "
", scopes: ["text.html.ruby"] waitsForPromise -> @@ -377,7 +344,7 @@ describe "TokenizedBuffer", -> runs -> fullyTokenize(tokenizedBuffer) - {tokens} = tokenizedBuffer.tokenizedLineForRow(0) + {tokens} = tokenizedBuffer.tokenizedLines[0] expect(tokens[0]).toEqual value: '<', scopes: ["text.html.ruby", "meta.tag.block.any.html", "punctuation.definition.tag.begin.html"] describe ".tokenForPosition(position)", -> @@ -387,11 +354,7 @@ describe "TokenizedBuffer", -> it "returns the correct token (regression)", -> buffer = atom.project.bufferForPathSync('sample.js') - tokenizedBuffer = new TokenizedBuffer({ - buffer, grammarRegistry: atom.grammars, packageManager: atom.packages, - assert: atom.assert, tabLength: 2, - }) - tokenizedBuffer.setGrammar(atom.grammars.grammarForScopeName('source.js')) + tokenizedBuffer = new TokenizedBuffer({buffer, grammar: atom.grammars.grammarForScopeName('source.js'), tabLength: 2}) fullyTokenize(tokenizedBuffer) expect(tokenizedBuffer.tokenForPosition([1, 0]).scopes).toEqual ["source.js"] expect(tokenizedBuffer.tokenForPosition([1, 1]).scopes).toEqual ["source.js"] @@ -400,16 +363,12 @@ describe "TokenizedBuffer", -> describe ".bufferRangeForScopeAtPosition(selector, position)", -> beforeEach -> buffer = atom.project.bufferForPathSync('sample.js') - tokenizedBuffer = new TokenizedBuffer({ - buffer, grammarRegistry: atom.grammars, packageManager: atom.packages, - assert: atom.assert, tabLength: 2, - }) - tokenizedBuffer.setGrammar(atom.grammars.grammarForScopeName('source.js')) + tokenizedBuffer = new TokenizedBuffer({buffer, grammar: atom.grammars.grammarForScopeName('source.js'), tabLength: 2}) fullyTokenize(tokenizedBuffer) describe "when the selector does not match the token at the position", -> it "returns a falsy value", -> - expect(tokenizedBuffer.bufferRangeForScopeAtPosition('.bogus', [0, 1])).toBeFalsy() + expect(tokenizedBuffer.bufferRangeForScopeAtPosition('.bogus', [0, 1])).toBeUndefined() describe "when the selector matches a single token at the position", -> it "returns the range covered by the token", -> @@ -423,11 +382,7 @@ describe "TokenizedBuffer", -> describe ".indentLevelForRow(row)", -> beforeEach -> buffer = atom.project.bufferForPathSync('sample.js') - tokenizedBuffer = new TokenizedBuffer({ - buffer, grammarRegistry: atom.grammars, packageManager: atom.packages, - assert: atom.assert, tabLength: 2, - }) - tokenizedBuffer.setGrammar(atom.grammars.grammarForScopeName('source.js')) + tokenizedBuffer = new TokenizedBuffer({buffer, grammar: atom.grammars.grammarForScopeName('source.js'), tabLength: 2}) fullyTokenize(tokenizedBuffer) describe "when the line is non-empty", -> @@ -469,7 +424,7 @@ describe "TokenizedBuffer", -> buffer.insert([12, 0], ' ') expect(tokenizedBuffer.indentLevelForRow(13)).toBe 2 - expect(tokenizedBuffer.tokenizedLineForRow(14)).not.toBeDefined() + expect(tokenizedBuffer.tokenizedLines[14]).not.toBeDefined() it "updates the indentLevel of empty lines surrounding a change that inserts lines", -> buffer.insert([7, 0], '\n\n') @@ -503,11 +458,7 @@ describe "TokenizedBuffer", -> buffer = atom.project.bufferForPathSync('sample.js') buffer.insert [10, 0], " // multi-line\n // comment\n // block\n" buffer.insert [0, 0], "// multi-line\n// comment\n// block\n" - tokenizedBuffer = new TokenizedBuffer({ - buffer, grammarRegistry: atom.grammars, packageManager: atom.packages, - assert: atom.assert, tabLength: 2, - }) - tokenizedBuffer.setGrammar(atom.grammars.grammarForScopeName('source.js')) + tokenizedBuffer = new TokenizedBuffer({buffer, grammar: atom.grammars.grammarForScopeName('source.js'), tabLength: 2}) fullyTokenize(tokenizedBuffer) it "includes the first line of multi-line comments", -> @@ -570,40 +521,69 @@ describe "TokenizedBuffer", -> expect(tokenizedBuffer.isFoldableAtRow(7)).toBe false expect(tokenizedBuffer.isFoldableAtRow(8)).toBe false + describe "::tokenizedLineForRow(row)", -> + it "returns the tokenized line for a row, or a placeholder line if it hasn't been tokenized yet", -> + buffer = atom.project.bufferForPathSync('sample.js') + grammar = atom.grammars.grammarForScopeName('source.js') + tokenizedBuffer = new TokenizedBuffer({buffer, grammar, tabLength: 2}) + line0 = buffer.lineForRow(0) + + jsScopeStartId = grammar.startIdForScope(grammar.scopeName) + jsScopeEndId = grammar.endIdForScope(grammar.scopeName) + startTokenizing(tokenizedBuffer) + expect(tokenizedBuffer.tokenizedLines[0]).toBeUndefined() + expect(tokenizedBuffer.tokenizedLineForRow(0).text).toBe(line0) + expect(tokenizedBuffer.tokenizedLineForRow(0).tags).toEqual([jsScopeStartId, line0.length, jsScopeEndId]) + advanceClock(1) + expect(tokenizedBuffer.tokenizedLines[0]).not.toBeUndefined() + expect(tokenizedBuffer.tokenizedLineForRow(0).text).toBe(line0) + expect(tokenizedBuffer.tokenizedLineForRow(0).tags).not.toEqual([jsScopeStartId, line0.length, jsScopeEndId]) + + nullScopeStartId = NullGrammar.startIdForScope(NullGrammar.scopeName) + nullScopeEndId = NullGrammar.endIdForScope(NullGrammar.scopeName) + tokenizedBuffer.setGrammar(NullGrammar) + startTokenizing(tokenizedBuffer) + expect(tokenizedBuffer.tokenizedLines[0]).toBeUndefined() + expect(tokenizedBuffer.tokenizedLineForRow(0).text).toBe(line0) + expect(tokenizedBuffer.tokenizedLineForRow(0).tags).toEqual([nullScopeStartId, line0.length, nullScopeEndId]) + advanceClock(1) + expect(tokenizedBuffer.tokenizedLineForRow(0).text).toBe(line0) + expect(tokenizedBuffer.tokenizedLineForRow(0).tags).toEqual([nullScopeStartId, line0.length, nullScopeEndId]) + + it "returns undefined if the requested row is outside the buffer range", -> + buffer = atom.project.bufferForPathSync('sample.js') + grammar = atom.grammars.grammarForScopeName('source.js') + tokenizedBuffer = new TokenizedBuffer({buffer, grammar, tabLength: 2}) + fullyTokenize(tokenizedBuffer) + expect(tokenizedBuffer.tokenizedLineForRow(999)).toBeUndefined() + describe "when the buffer is configured with the null grammar", -> - it "uses the placeholder tokens and does not actually tokenize using the grammar", -> - spyOn(atom.grammars.nullGrammar, 'tokenizeLine').andCallThrough() + it "does not actually tokenize using the grammar", -> + spyOn(NullGrammar, 'tokenizeLine').andCallThrough() buffer = atom.project.bufferForPathSync('sample.will-use-the-null-grammar') buffer.setText('a\nb\nc') - - tokenizedBuffer = new TokenizedBuffer({ - buffer, grammarRegistry: atom.grammars, packageManager: atom.packages, - assert: atom.assert, tabLength: 2, - }) + tokenizedBuffer = new TokenizedBuffer({buffer, tabLength: 2}) tokenizeCallback = jasmine.createSpy('onDidTokenize') tokenizedBuffer.onDidTokenize(tokenizeCallback) + expect(tokenizedBuffer.tokenizedLines[0]).toBeUndefined() + expect(tokenizedBuffer.tokenizedLines[1]).toBeUndefined() + expect(tokenizedBuffer.tokenizedLines[2]).toBeUndefined() + expect(tokenizeCallback.callCount).toBe(0) + expect(NullGrammar.tokenizeLine).not.toHaveBeenCalled() + fullyTokenize(tokenizedBuffer) - - expect(tokenizeCallback.callCount).toBe 1 - expect(atom.grammars.nullGrammar.tokenizeLine.callCount).toBe 0 - - expect(tokenizedBuffer.tokenizedLineForRow(0).tokens.length).toBe 1 - expect(tokenizedBuffer.tokenizedLineForRow(0).tokens[0].value).toBe 'a' - expect(tokenizedBuffer.tokenizedLineForRow(1).tokens.length).toBe 1 - expect(tokenizedBuffer.tokenizedLineForRow(1).tokens[0].value).toBe 'b' - expect(tokenizedBuffer.tokenizedLineForRow(2).tokens.length).toBe 1 - expect(tokenizedBuffer.tokenizedLineForRow(2).tokens[0].value).toBe 'c' + expect(tokenizedBuffer.tokenizedLines[0]).toBeUndefined() + expect(tokenizedBuffer.tokenizedLines[1]).toBeUndefined() + expect(tokenizedBuffer.tokenizedLines[2]).toBeUndefined() + expect(tokenizeCallback.callCount).toBe(0) + expect(NullGrammar.tokenizeLine).not.toHaveBeenCalled() describe "text decoration layer API", -> describe "iterator", -> it "iterates over the syntactic scope boundaries", -> buffer = new TextBuffer(text: "var foo = 1 /*\nhello*/var bar = 2\n") - tokenizedBuffer = new TokenizedBuffer({ - buffer, grammarRegistry: atom.grammars, packageManager: atom.packages, - assert: atom.assert, tabLength: 2, - }) - tokenizedBuffer.setGrammar(atom.grammars.selectGrammar(".js")) + tokenizedBuffer = new TokenizedBuffer({buffer, grammar: atom.grammars.grammarForScopeName("source.js"), tabLength: 2}) fullyTokenize(tokenizedBuffer) iterator = tokenizedBuffer.buildIterator() @@ -655,11 +635,7 @@ describe "TokenizedBuffer", -> runs -> buffer = new TextBuffer(text: "# hello\n# world") - tokenizedBuffer = new TokenizedBuffer({ - buffer, grammarRegistry: atom.grammars, packageManager: atom.packages, - assert: atom.assert, tabLength: 2, - }) - tokenizedBuffer.setGrammar(atom.grammars.selectGrammar(".coffee")) + tokenizedBuffer = new TokenizedBuffer({buffer, grammar: atom.grammars.grammarForScopeName("source.coffee"), tabLength: 2}) fullyTokenize(tokenizedBuffer) iterator = tokenizedBuffer.buildIterator() @@ -688,11 +664,7 @@ describe "TokenizedBuffer", -> }) buffer = new TextBuffer(text: 'start x\nend x\nx') - tokenizedBuffer = new TokenizedBuffer({ - buffer, grammarRegistry: atom.grammars, packageManager: atom.packages, - assert: atom.assert, tabLength: 2, - }) - tokenizedBuffer.setGrammar(grammar) + tokenizedBuffer = new TokenizedBuffer({buffer, grammar, tabLength: 2}) fullyTokenize(tokenizedBuffer) iterator = tokenizedBuffer.buildIterator() diff --git a/src/language-mode.coffee b/src/language-mode.coffee index ad038d7db..bb9f339c4 100644 --- a/src/language-mode.coffee +++ b/src/language-mode.coffee @@ -148,19 +148,19 @@ class LanguageMode rowRange rowRangeForCommentAtBufferRow: (bufferRow) -> - return unless @editor.tokenizedBuffer.tokenizedLineForRow(bufferRow).isComment() + return unless @editor.tokenizedBuffer.tokenizedLines[bufferRow]?.isComment() startRow = bufferRow endRow = bufferRow if bufferRow > 0 for currentRow in [bufferRow-1..0] by -1 - break unless @editor.tokenizedBuffer.tokenizedLineForRow(currentRow).isComment() + break unless @editor.tokenizedBuffer.tokenizedLines[currentRow]?.isComment() startRow = currentRow if bufferRow < @buffer.getLastRow() for currentRow in [bufferRow+1..@buffer.getLastRow()] by 1 - break unless @editor.tokenizedBuffer.tokenizedLineForRow(currentRow).isComment() + break unless @editor.tokenizedBuffer.tokenizedLines[currentRow]?.isComment() endRow = currentRow return [startRow, endRow] if startRow isnt endRow @@ -189,7 +189,7 @@ class LanguageMode # row is a comment. isLineCommentedAtBufferRow: (bufferRow) -> return false unless 0 <= bufferRow <= @editor.getLastBufferRow() - @editor.tokenizedBuffer.tokenizedLineForRow(bufferRow).isComment() + @editor.tokenizedBuffer.tokenizedLines[bufferRow]?.isComment() # Find a row range for a 'paragraph' around specified bufferRow. A paragraph # is a block of text bounded by and empty line or a block of text that is not @@ -246,10 +246,7 @@ class LanguageMode @suggestedIndentForTokenizedLineAtBufferRow(bufferRow, line, tokenizedLine, options) suggestedIndentForLineAtBufferRow: (bufferRow, line, options) -> - if @editor.largeFileMode or @editor.tokenizedBuffer.grammar is NullGrammar - tokenizedLine = @editor.tokenizedBuffer.buildPlaceholderTokenizedLineForRowWithText(bufferRow, line) - else - tokenizedLine = @editor.tokenizedBuffer.buildTokenizedLineForRowWithText(bufferRow, line) + tokenizedLine = @editor.tokenizedBuffer.buildTokenizedLineForRowWithText(bufferRow, line) @suggestedIndentForTokenizedLineAtBufferRow(bufferRow, line, tokenizedLine, options) suggestedIndentForTokenizedLineAtBufferRow: (bufferRow, line, tokenizedLine, options) -> diff --git a/src/null-grammar.js b/src/null-grammar.js index 0ca3f83f1..fe9c3889e 100644 --- a/src/null-grammar.js +++ b/src/null-grammar.js @@ -2,12 +2,39 @@ import {Disposable} from 'event-kit' -export default Object.freeze({ +export default { name: 'Null Grammar', - scopeName: 'text.plain', + scopeName: 'text.plain.null-grammar', + scopeForId (id) { + if (id === -1 || id === -2) { + return this.scopeName + } else { + return null + } + }, + startIdForScope (scopeName) { + if (scopeName === this.scopeName) { + return -1 + } else { + return null + } + }, + endIdForScope (scopeName) { + if (scopeName === this.scopeName) { + return -2 + } else { + return null + } + }, + tokenizeLine (text) { + return { + tags: [this.startIdForScope(this.scopeName), text.length, this.endIdForScope(this.scopeName)], + ruleStack: null + } + }, onDidUpdate (callback) { return new Disposable(noop) } -}) +} function noop () {} diff --git a/src/text-editor.coffee b/src/text-editor.coffee index 02cce3daf..50b2e6f96 100644 --- a/src/text-editor.coffee +++ b/src/text-editor.coffee @@ -2868,7 +2868,7 @@ class TextEditor extends Model # whitespace. usesSoftTabs: -> for bufferRow in [0..@buffer.getLastRow()] - continue if @tokenizedBuffer.tokenizedLineForRow(bufferRow).isComment() + continue if @tokenizedBuffer.tokenizedLines[bufferRow]?.isComment() line = @buffer.lineForRow(bufferRow) return true if line[0] is ' ' diff --git a/src/tokenized-buffer.coffee b/src/tokenized-buffer.coffee index 80358f23d..ce56e0388 100644 --- a/src/tokenized-buffer.coffee +++ b/src/tokenized-buffer.coffee @@ -36,7 +36,6 @@ class TokenizedBuffer extends Model @tokenIterator = new TokenIterator(this) @disposables.add @buffer.registerTextDecorationLayer(this) - @rootScopeDescriptor = new ScopeDescriptor(scopes: ['text.plain']) @setGrammar(grammar ? NullGrammar) @@ -95,14 +94,17 @@ class TokenizedBuffer extends Model false retokenizeLines: -> - lastRow = @buffer.getLastRow() @fullyTokenized = false - @tokenizedLines = new Array(lastRow + 1) + @tokenizedLines = new Array(@buffer.getLineCount()) @invalidRows = [] - @invalidateRow(0) + if @largeFileMode or @grammar.name is 'Null Grammar' + @markTokenizationComplete() + else + @invalidateRow(0) setVisible: (@visible) -> - @tokenizeInBackground() if @visible + if @visible and @grammar.name isnt 'Null Grammar' and not @largeFileMode + @tokenizeInBackground() getTabLength: -> @tabLength @@ -117,12 +119,6 @@ class TokenizedBuffer extends Model @tokenizeNextChunk() if @isAlive() and @buffer.isAlive() tokenizeNextChunk: -> - # Short circuit null grammar which can just use the placeholder tokens - if (@grammar.name is 'Null Grammar') and @firstInvalidRow()? - @invalidRows = [] - @markTokenizationComplete() - return - rowsRemaining = @chunkSize while @firstInvalidRow()? and rowsRemaining > 0 @@ -167,8 +163,6 @@ class TokenizedBuffer extends Model return invalidateRow: (row) -> - return if @largeFileMode - @invalidRows.push(row) @invalidRows.sort (a, b) -> a - b @tokenizeInBackground() @@ -189,18 +183,19 @@ class TokenizedBuffer extends Model start = oldRange.start.row end = oldRange.end.row delta = newRange.end.row - oldRange.end.row + oldLineCount = oldRange.end.row - oldRange.start.row + 1 + newLineCount = newRange.end.row - newRange.start.row + 1 @updateInvalidRows(start, end, delta) previousEndStack = @stackForRow(end) # used in spill detection below - if @largeFileMode or @grammar is NullGrammar - newTokenizedLines = @buildPlaceholderTokenizedLinesForRows(start, end + delta) + if @largeFileMode or @grammar.name is 'Null Grammar' + _.spliceWithArray(@tokenizedLines, start, oldLineCount, new Array(newLineCount)) else newTokenizedLines = @buildTokenizedLinesForRows(start, end + delta, @stackForRow(start - 1), @openScopesForRow(start)) - _.spliceWithArray(@tokenizedLines, start, end - start + 1, newTokenizedLines) - - newEndStack = @stackForRow(end + delta) - if newEndStack and not _.isEqual(newEndStack, previousEndStack) - @invalidateRow(end + delta + 1) + _.spliceWithArray(@tokenizedLines, start, oldLineCount, newTokenizedLines) + newEndStack = @stackForRow(end + delta) + if newEndStack and not _.isEqual(newEndStack, previousEndStack) + @invalidateRow(end + delta + 1) isFoldableAtRow: (row) -> if @largeFileMode @@ -211,46 +206,39 @@ class TokenizedBuffer extends Model # Returns a {Boolean} indicating whether the given buffer row starts # a a foldable row range due to the code's indentation patterns. isFoldableCodeAtRow: (row) -> - # Investigating an exception that's occurring here due to the line being - # undefined. This should paper over the problem but we want to figure out - # what is happening: - tokenizedLine = @tokenizedLineForRow(row) - @assert tokenizedLine?, "TokenizedLine is undefined", (error) => - error.metadata = { - row: row - rowCount: @tokenizedLines.length - tokenizedBufferChangeCount: @changeCount - bufferChangeCount: @buffer.changeCount - } - - return false unless tokenizedLine? - - return false if @buffer.isRowBlank(row) or tokenizedLine.isComment() - nextRow = @buffer.nextNonBlankRow(row) - return false unless nextRow? - - @indentLevelForRow(nextRow) > @indentLevelForRow(row) + if 0 <= row <= @buffer.getLastRow() + nextRow = @buffer.nextNonBlankRow(row) + tokenizedLine = @tokenizedLines[row] + if @buffer.isRowBlank(row) or tokenizedLine?.isComment() or not nextRow? + false + else + @indentLevelForRow(nextRow) > @indentLevelForRow(row) + else + false isFoldableCommentAtRow: (row) -> previousRow = row - 1 nextRow = row + 1 - return false if nextRow > @buffer.getLastRow() - - (row is 0 or not @tokenizedLineForRow(previousRow).isComment()) and - @tokenizedLineForRow(row).isComment() and - @tokenizedLineForRow(nextRow).isComment() + if nextRow > @buffer.getLastRow() + false + else + Boolean( + not (@tokenizedLines[previousRow]?.isComment()) and + @tokenizedLines[row]?.isComment() and + @tokenizedLines[nextRow]?.isComment() + ) buildTokenizedLinesForRows: (startRow, endRow, startingStack, startingopenScopes) -> ruleStack = startingStack openScopes = startingopenScopes stopTokenizingAt = startRow + @chunkSize - tokenizedLines = for row in [startRow..endRow] + tokenizedLines = for row in [startRow..endRow] by 1 if (ruleStack or row is 0) and row < stopTokenizingAt tokenizedLine = @buildTokenizedLineForRow(row, ruleStack, openScopes) ruleStack = tokenizedLine.ruleStack openScopes = @scopesFromTags(openScopes, tokenizedLine.tags) else - tokenizedLine = @buildPlaceholderTokenizedLineForRow(row, openScopes) + tokenizedLine = undefined tokenizedLine if endRow >= stopTokenizingAt @@ -259,21 +247,6 @@ class TokenizedBuffer extends Model tokenizedLines - buildPlaceholderTokenizedLinesForRows: (startRow, endRow) -> - @buildPlaceholderTokenizedLineForRow(row) for row in [startRow..endRow] by 1 - - buildPlaceholderTokenizedLineForRow: (row) -> - @buildPlaceholderTokenizedLineForRowWithText(row, @buffer.lineForRow(row)) - - buildPlaceholderTokenizedLineForRowWithText: (row, text) -> - if @grammar isnt NullGrammar - openScopes = [@grammar.startIdForScope(@grammar.scopeName)] - else - openScopes = [] - tags = [text.length] - lineEnding = @buffer.lineEndingForRow(row) - new TokenizedLine({openScopes, text, tags, lineEnding, @tokenIterator}) - buildTokenizedLineForRow: (row, ruleStack, openScopes) -> @buildTokenizedLineForRowWithText(row, @buffer.lineForRow(row), ruleStack, openScopes) @@ -283,8 +256,14 @@ class TokenizedBuffer extends Model new TokenizedLine({openScopes, text, tags, ruleStack, lineEnding, @tokenIterator}) tokenizedLineForRow: (bufferRow) -> - if 0 <= bufferRow < @tokenizedLines.length - @tokenizedLines[bufferRow] ?= @buildPlaceholderTokenizedLineForRow(bufferRow) + if 0 <= bufferRow <= @buffer.getLastRow() + if tokenizedLine = @tokenizedLines[bufferRow] + tokenizedLine + else + text = @buffer.lineForRow(bufferRow) + lineEnding = @buffer.lineEndingForRow(bufferRow) + tags = [@grammar.startIdForScope(@grammar.scopeName), text.length, @grammar.endIdForScope(@grammar.scopeName)] + @tokenizedLines[bufferRow] = new TokenizedLine({openScopes: [], text, tags, lineEnding, @tokenIterator}) tokenizedLinesForRows: (startRow, endRow) -> for row in [startRow..endRow] by 1 @@ -294,8 +273,7 @@ class TokenizedBuffer extends Model @tokenizedLines[bufferRow]?.ruleStack openScopesForRow: (bufferRow) -> - if bufferRow > 0 - precedingLine = @tokenizedLineForRow(bufferRow - 1) + if precedingLine = @tokenizedLines[bufferRow - 1] @scopesFromTags(precedingLine.openScopes, precedingLine.tags) else [] @@ -448,7 +426,7 @@ class TokenizedBuffer extends Model logLines: (start=0, end=@buffer.getLastRow()) -> for row in [start..end] - line = @tokenizedLineForRow(row).text + line = @tokenizedLines[row].text console.log row, line, line.length return