diff --git a/spec/sample-with-comments.js b/spec/sample-with-comments.js deleted file mode 100644 index 66dc9051d..000000000 --- a/spec/sample-with-comments.js +++ /dev/null @@ -1 +0,0 @@ -undefined \ No newline at end of file diff --git a/spec/text-editor-registry-spec.js b/spec/text-editor-registry-spec.js index 4f4d1ee93..4c6d680eb 100644 --- a/spec/text-editor-registry-spec.js +++ b/spec/text-editor-registry-spec.js @@ -1,6 +1,7 @@ const TextEditorRegistry = require('../src/text-editor-registry') const TextEditor = require('../src/text-editor') const TextBuffer = require('text-buffer') +const {Point, Range} = TextBuffer const {it, fit, ffit, fffit} = require('./async-spec-helpers') const dedent = require('dedent') @@ -257,19 +258,19 @@ describe('TextEditorRegistry', function () { describe('when the "tabType" config setting is "auto"', function () { it('enables or disables soft tabs based on the editor\'s content', async function () { + await initialPackageActivation await atom.packages.activatePackage('language-javascript') atom.grammars.assignLanguageMode(editor, 'source.js') atom.config.set('editor.tabType', 'auto') - - registry.maintainConfig(editor) await initialPackageActivation + const languageMode = editor.getBuffer().getLanguageMode() editor.setText(dedent` { hello; } `) - editor.getBuffer().getLanguageMode().retokenizeLines() + let disposable = registry.maintainConfig(editor) expect(editor.getSoftTabs()).toBe(true) editor.setText(dedent` @@ -277,18 +278,17 @@ describe('TextEditorRegistry', function () { hello; } `) - editor.getBuffer().getLanguageMode().retokenizeLines() + disposable.dispose() + disposable = registry.maintainConfig(editor) expect(editor.getSoftTabs()).toBe(false) - editor.setText(dedent` + editor.setTextInBufferRange(new Range(Point.ZERO, Point.ZERO), dedent` /* * Comment with a leading space. */ - { - ${'\t'}hello; - } - ` + editor.getText()) - editor.getBuffer().getLanguageMode().retokenizeLines() + ` + '\n') + disposable.dispose() + disposable = registry.maintainConfig(editor) expect(editor.getSoftTabs()).toBe(false) editor.setText(dedent` @@ -300,8 +300,8 @@ describe('TextEditorRegistry', function () { hello; } `) - - editor.getBuffer().getLanguageMode().retokenizeLines() + disposable.dispose() + disposable = registry.maintainConfig(editor) expect(editor.getSoftTabs()).toBe(false) editor.setText(dedent` @@ -313,7 +313,8 @@ describe('TextEditorRegistry', function () { hello; } `) - editor.getBuffer().getLanguageMode().retokenizeLines() + disposable.dispose() + disposable = registry.maintainConfig(editor) expect(editor.getSoftTabs()).toBe(true) }) }) diff --git a/spec/tree-sitter-language-mode-spec.js b/spec/tree-sitter-language-mode-spec.js index 01b053e35..58dae0241 100644 --- a/spec/tree-sitter-language-mode-spec.js +++ b/spec/tree-sitter-language-mode-spec.js @@ -39,7 +39,6 @@ describe('TreeSitterLanguageMode', () => { const languageMode = new TreeSitterLanguageMode({buffer, grammar}) buffer.setLanguageMode(languageMode) - await nextHighlightingUpdate(languageMode) expectTokensToEqual(editor, [[ {text: 'aa.', scopes: ['source']}, @@ -69,7 +68,6 @@ describe('TreeSitterLanguageMode', () => { const languageMode = new TreeSitterLanguageMode({buffer, grammar}) buffer.setLanguageMode(languageMode) - await nextHighlightingUpdate(languageMode) expectTokensToEqual(editor, [[ {text: 'a', scopes: ['source', 'variable']}, @@ -96,7 +94,6 @@ describe('TreeSitterLanguageMode', () => { const languageMode = new TreeSitterLanguageMode({buffer, grammar}) buffer.setLanguageMode(languageMode) - await nextHighlightingUpdate(languageMode) expectTokensToEqual(editor, [ [ @@ -124,7 +121,6 @@ describe('TreeSitterLanguageMode', () => { const languageMode = new TreeSitterLanguageMode({buffer, grammar}) buffer.setLanguageMode(languageMode) - await nextHighlightingUpdate(languageMode) expect( languageMode.tree.rootNode.descendantForPosition(Point(1, 2), Point(1, 6)).toString() @@ -169,7 +165,6 @@ describe('TreeSitterLanguageMode', () => { const languageMode = new TreeSitterLanguageMode({buffer, grammar}) buffer.setLanguageMode(languageMode) - await nextHighlightingUpdate(languageMode) // missing closing paren expectTokensToEqual(editor, [ @@ -208,7 +203,6 @@ describe('TreeSitterLanguageMode', () => { const languageMode = new TreeSitterLanguageMode({buffer, grammar}) buffer.setLanguageMode(languageMode) - await nextHighlightingUpdate(languageMode) expectTokensToEqual(editor, [ [ @@ -241,7 +235,6 @@ describe('TreeSitterLanguageMode', () => { const languageMode = new TreeSitterLanguageMode({buffer, grammar}) buffer.setLanguageMode(languageMode) - await nextHighlightingUpdate(languageMode) expectTokensToEqual(editor, [ [{text: '// abc', scopes: ['comment']}], [{text: '', scopes: []}], @@ -254,7 +247,6 @@ describe('TreeSitterLanguageMode', () => { ]) buffer.insert([2, 0], ' ') - await nextHighlightingUpdate(languageMode) expectTokensToEqual(editor, [ [{text: '// abc', scopes: ['comment']}], [{text: '', scopes: []}], @@ -282,7 +274,6 @@ describe('TreeSitterLanguageMode', () => { const languageMode = new TreeSitterLanguageMode({buffer, grammar}) buffer.setLanguageMode(languageMode) - await nextHighlightingUpdate(languageMode) expectTokensToEqual(editor, [ [ @@ -324,7 +315,6 @@ describe('TreeSitterLanguageMode', () => { const languageMode = new TreeSitterLanguageMode({buffer, grammar}) buffer.setLanguageMode(languageMode) - await nextHighlightingUpdate(languageMode) editor.foldBufferRange([[0, 2], [2, 0]]) @@ -361,7 +351,6 @@ describe('TreeSitterLanguageMode', () => { const languageMode = new TreeSitterLanguageMode({buffer, grammar}) buffer.setLanguageMode(languageMode) - await nextHighlightingUpdate(languageMode) expectTokensToEqual(editor, [ [ @@ -394,7 +383,6 @@ describe('TreeSitterLanguageMode', () => { const languageMode = new TreeSitterLanguageMode({buffer, grammar}) buffer.setLanguageMode(languageMode) - await nextHighlightingUpdate(languageMode) expectTokensToEqual(editor, [ [ @@ -422,7 +410,7 @@ describe('TreeSitterLanguageMode', () => { buffer.setText('abc;'); - const languageMode = new TreeSitterLanguageMode({buffer, grammar}) + const languageMode = new TreeSitterLanguageMode({buffer, grammar, syncOperationLimit: 0}) buffer.setLanguageMode(languageMode) await nextHighlightingUpdate(languageMode) await new Promise(process.nextTick) @@ -509,8 +497,6 @@ describe('TreeSitterLanguageMode', () => { const languageMode = new TreeSitterLanguageMode({buffer, grammar: jsGrammar, grammars: atom.grammars}) buffer.setLanguageMode(languageMode) - await nextHighlightingUpdate(languageMode) - await nextHighlightingUpdate(languageMode) expectTokensToEqual(editor, [ [ @@ -570,8 +556,6 @@ describe('TreeSitterLanguageMode', () => { const languageMode = new TreeSitterLanguageMode({buffer, grammar: htmlGrammar, grammars: atom.grammars}) buffer.setLanguageMode(languageMode) - await nextHighlightingUpdate(languageMode) - await nextHighlightingUpdate(languageMode) expectTokensToEqual(editor, [ [ @@ -608,7 +592,6 @@ describe('TreeSitterLanguageMode', () => { const languageMode = new TreeSitterLanguageMode({buffer, grammar: jsGrammar, grammars: atom.grammars}) buffer.setLanguageMode(languageMode) - await nextHighlightingUpdate(languageMode) expectTokensToEqual(editor, [ [ {text: 'node.', scopes: []}, @@ -683,14 +666,13 @@ describe('TreeSitterLanguageMode', () => { atom.grammars.addGrammar(htmlGrammar) buffer.setText('\n\n') - const languageMode = new TreeSitterLanguageMode({buffer, grammar: ejsGrammar, grammars: atom.grammars}) + const languageMode = new TreeSitterLanguageMode({ + buffer, + grammar: ejsGrammar, + grammars: atom.grammars, + }) buffer.setLanguageMode(languageMode) - // Parse EJS, then HTML and template JS in parallel, then script tag JS - await nextHighlightingUpdate(languageMode) - await nextHighlightingUpdate(languageMode) - await nextHighlightingUpdate(languageMode) - expectTokensToEqual(editor, [ [ {text: '<', scopes: ['html']}, @@ -752,7 +734,12 @@ describe('TreeSitterLanguageMode', () => { atom.grammars.addGrammar(htmlGrammar) buffer.setText('') - const languageMode = new TreeSitterLanguageMode({buffer, grammar: htmlGrammar, grammars: atom.grammars}) + const languageMode = new TreeSitterLanguageMode({ + buffer, + grammar: htmlGrammar, + grammars: atom.grammars, + syncOperationLimit: 0 + }) buffer.setLanguageMode(languageMode) await promise @@ -789,7 +776,6 @@ describe('TreeSitterLanguageMode', () => { const languageMode = new TreeSitterLanguageMode({buffer, grammar}) buffer.setLanguageMode(languageMode) - await nextHighlightingUpdate(languageMode) expect(editor.isFoldableAtBufferRow(0)).toBe(false) expect(editor.isFoldableAtBufferRow(1)).toBe(true) @@ -844,7 +830,6 @@ describe('TreeSitterLanguageMode', () => { const languageMode = new TreeSitterLanguageMode({buffer, grammar}) buffer.setLanguageMode(languageMode) - await nextHighlightingUpdate(languageMode) // Avoid bringing the `else if...` up onto the same screen line as the preceding `if`. editor.foldBufferRow(1) @@ -900,7 +885,6 @@ describe('TreeSitterLanguageMode', () => { const languageMode = new TreeSitterLanguageMode({buffer, grammar}) buffer.setLanguageMode(languageMode) - await nextHighlightingUpdate(languageMode) expect(editor.isFoldableAtBufferRow(0)).toBe(true) expect(editor.isFoldableAtBufferRow(1)).toBe(false) @@ -949,7 +933,6 @@ describe('TreeSitterLanguageMode', () => { const languageMode = new TreeSitterLanguageMode({buffer, grammar}) buffer.setLanguageMode(languageMode) - await nextHighlightingUpdate(languageMode) expect(editor.isFoldableAtBufferRow(0)).toBe(true) expect(editor.isFoldableAtBufferRow(1)).toBe(false) @@ -1024,7 +1007,6 @@ describe('TreeSitterLanguageMode', () => { const languageMode = new TreeSitterLanguageMode({buffer, grammar}) buffer.setLanguageMode(languageMode) - await nextHighlightingUpdate(languageMode) editor.foldBufferRow(3) expect(getDisplayText(editor)).toBe(dedent ` @@ -1106,7 +1088,6 @@ describe('TreeSitterLanguageMode', () => { const languageMode = new TreeSitterLanguageMode({buffer, grammar}) buffer.setLanguageMode(languageMode) - await nextHighlightingUpdate(languageMode) // Void elements have only one child expect(editor.isFoldableAtBufferRow(1)).toBe(false) @@ -1159,7 +1140,6 @@ describe('TreeSitterLanguageMode', () => { const languageMode = new TreeSitterLanguageMode({buffer, grammar}) buffer.setLanguageMode(languageMode) - await nextHighlightingUpdate(languageMode) expect(languageMode.tree.rootNode.toString()).toBe( "(program (if (identifier) " + @@ -1229,7 +1209,6 @@ describe('TreeSitterLanguageMode', () => { `) buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar})) - await nextHighlightingUpdate(buffer.getLanguageMode()) editor.foldBufferRow(0) expect(getDisplayText(editor)).toBe(dedent ` @@ -1289,9 +1268,6 @@ describe('TreeSitterLanguageMode', () => { const languageMode = new TreeSitterLanguageMode({buffer, grammar: jsGrammar, grammars: atom.grammars}) buffer.setLanguageMode(languageMode) - await nextHighlightingUpdate(languageMode) - await nextHighlightingUpdate(languageMode) - editor.foldBufferRow(2) expect(getDisplayText(editor)).toBe( `a = html \` @@ -1334,7 +1310,6 @@ describe('TreeSitterLanguageMode', () => { buffer.setText('foo({bar: baz});') buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar})) - await nextHighlightingUpdate(buffer.getLanguageMode()) expect(editor.scopeDescriptorForBufferPosition([0, 'foo({b'.length]).getScopesArray()).toEqual([ 'source.js', 'property.name' @@ -1385,9 +1360,6 @@ describe('TreeSitterLanguageMode', () => { const languageMode = new TreeSitterLanguageMode({buffer, grammar: htmlGrammar, grammars: atom.grammars}) buffer.setLanguageMode(languageMode) - await nextHighlightingUpdate(languageMode) - await nextHighlightingUpdate(languageMode) - await nextHighlightingUpdate(languageMode) const position = buffer.findSync('name').start expect(languageMode.scopeDescriptorForPosition(position).getScopesArray()).toEqual([ @@ -1412,7 +1384,6 @@ describe('TreeSitterLanguageMode', () => { buffer.setText('foo({bar: baz});') buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar})) - await nextHighlightingUpdate(buffer.getLanguageMode()) expect(editor.bufferRangeForScopeAtPosition(null, [0, 6])).toEqual( [[0, 5], [0, 8]] ) @@ -1453,9 +1424,6 @@ describe('TreeSitterLanguageMode', () => { const languageMode = new TreeSitterLanguageMode({buffer, grammar: htmlGrammar, grammars: atom.grammars}) buffer.setLanguageMode(languageMode) - await nextHighlightingUpdate(languageMode) - await nextHighlightingUpdate(languageMode) - await nextHighlightingUpdate(languageMode) const nameProperty = buffer.findSync('name') const {start} = nameProperty @@ -1475,7 +1443,6 @@ describe('TreeSitterLanguageMode', () => { buffer.setText('foo({bar: baz});') buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar})) - await nextHighlightingUpdate(buffer.getLanguageMode()) expect(editor.bufferRangeForScopeAtPosition('.property_identifier', [0, 6])).toEqual( buffer.findSync('bar') ) @@ -1519,9 +1486,6 @@ describe('TreeSitterLanguageMode', () => { const languageMode = new TreeSitterLanguageMode({buffer, grammar: htmlGrammar, grammars: atom.grammars}) buffer.setLanguageMode(languageMode) - await nextHighlightingUpdate(languageMode) - await nextHighlightingUpdate(languageMode) - await nextHighlightingUpdate(languageMode) const nameProperty = buffer.findSync('name') const {start} = nameProperty @@ -1564,9 +1528,6 @@ describe('TreeSitterLanguageMode', () => { const languageMode = new TreeSitterLanguageMode({buffer, grammar: htmlGrammar, grammars: atom.grammars}) buffer.setLanguageMode(languageMode) - await nextHighlightingUpdate(languageMode) - await nextHighlightingUpdate(languageMode) - await nextHighlightingUpdate(languageMode) const nameProperty = buffer.findSync('name') const {start} = nameProperty @@ -1589,7 +1550,6 @@ describe('TreeSitterLanguageMode', () => { buffer.setText('foo(bar({x: 2}));') const languageMode = new TreeSitterLanguageMode({buffer, grammar}) buffer.setLanguageMode(languageMode) - await nextHighlightingUpdate(languageMode) expect(languageMode.getSyntaxNodeAtPosition([0, 6]).range).toEqual( buffer.findSync('bar') ) @@ -1616,7 +1576,6 @@ describe('TreeSitterLanguageMode', () => { `) buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar})) - await nextHighlightingUpdate(buffer.getLanguageMode()) editor.setCursorBufferPosition([1, 3]) editor.selectLargerSyntaxNode() @@ -1674,9 +1633,6 @@ describe('TreeSitterLanguageMode', () => { const languageMode = new TreeSitterLanguageMode({buffer, grammar: jsGrammar, grammars: atom.grammars}) buffer.setLanguageMode(languageMode) - await nextHighlightingUpdate(languageMode) - await nextHighlightingUpdate(languageMode) - editor.setCursorBufferPosition({row: 0, column: buffer.getText().indexOf('ef()')}) editor.selectLargerSyntaxNode() expect(editor.getSelectedText()).toBe('def') diff --git a/src/grammar-registry.js b/src/grammar-registry.js index d82f14cb1..0c79d5b1e 100644 --- a/src/grammar-registry.js +++ b/src/grammar-registry.js @@ -134,7 +134,7 @@ class GrammarRegistry { } this.grammarScoresByBuffer.set(buffer, null) - if (grammar.scopeName !== buffer.getLanguageMode().getLanguageId()) { + if (grammar !== buffer.getLanguageMode().grammar) { buffer.setLanguageMode(this.languageModeForGrammarAndBuffer(grammar, buffer)) } @@ -161,7 +161,7 @@ class GrammarRegistry { ) this.languageOverridesByBufferId.delete(buffer.id) this.grammarScoresByBuffer.set(buffer, result.score) - if (result.grammar.scopeName !== buffer.getLanguageMode().getLanguageId()) { + if (result.grammar !== buffer.getLanguageMode().grammar) { buffer.setLanguageMode(this.languageModeForGrammarAndBuffer(result.grammar, buffer)) } } diff --git a/src/text-editor-registry.js b/src/text-editor-registry.js index 132b24ffb..e9dbf2f5c 100644 --- a/src/text-editor-registry.js +++ b/src/text-editor-registry.js @@ -218,7 +218,7 @@ class TextEditorRegistry { async updateAndMonitorEditorSettings (editor, oldLanguageMode) { await this.initialPackageActivationPromise this.updateEditorSettingsForLanguageMode(editor, oldLanguageMode) - await this.subscribeToSettingsForEditorScope(editor) + this.subscribeToSettingsForEditorScope(editor) } updateEditorSettingsForLanguageMode (editor, oldLanguageMode) { @@ -246,7 +246,9 @@ class TextEditorRegistry { } } - async subscribeToSettingsForEditorScope (editor) { + subscribeToSettingsForEditorScope (editor) { + if (!this.editorsWithMaintainedConfig) return + const scopeDescriptor = editor.getRootScopeDescriptor() const scopeChain = scopeDescriptor.getScopeChain() diff --git a/src/text-editor.js b/src/text-editor.js index ba063f7f0..3616db28c 100644 --- a/src/text-editor.js +++ b/src/text-editor.js @@ -4843,7 +4843,7 @@ class TextEditor { let endRow = bufferRow const rowCount = this.getLineCount() - while (endRow < rowCount) { + while (endRow + 1 < rowCount) { if (!NON_WHITESPACE_REGEXP.test(this.lineTextForBufferRow(endRow + 1))) break if (languageMode.isRowCommented(endRow + 1) !== isCommented) break endRow++ diff --git a/src/tree-sitter-language-mode.js b/src/tree-sitter-language-mode.js index c046cf776..103b8816e 100644 --- a/src/tree-sitter-language-mode.js +++ b/src/tree-sitter-language-mode.js @@ -23,7 +23,7 @@ class TreeSitterLanguageMode { } } - constructor ({buffer, grammar, config, grammars}) { + constructor ({buffer, grammar, config, grammars, syncOperationLimit}) { TreeSitterLanguageMode._patchSyntaxNode() this.id = nextId++ this.buffer = buffer @@ -34,6 +34,10 @@ class TreeSitterLanguageMode { this.rootLanguageLayer = new LanguageLayer(this, grammar) this.injectionsMarkerLayer = buffer.addMarkerLayer() + if (syncOperationLimit != null) { + this.syncOperationLimit = syncOperationLimit + } + this.rootScopeDescriptor = new ScopeDescriptor({scopes: [this.grammar.scopeName]}) this.emitter = new Emitter() this.isFoldableCache = [] @@ -83,15 +87,23 @@ class TreeSitterLanguageMode { } } - async parse (language, oldTree, ranges) { + parse (language, oldTree, ranges) { const parser = PARSER_POOL.pop() || new Parser() parser.setLanguage(language) - const newTree = await parser.parseTextBuffer(this.buffer.buffer, oldTree, { - syncOperationLimit: 1000, + const result = parser.parseTextBuffer(this.buffer.buffer, oldTree, { + syncOperationLimit: this.syncOperationLimit, includedRanges: ranges }) - PARSER_POOL.push(parser) - return newTree + + if (result.then) { + return result.then(tree => { + PARSER_POOL.push(parser) + return tree + }) + } else { + PARSER_POOL.push(parser) + return result + } } get tree () { @@ -107,6 +119,7 @@ class TreeSitterLanguageMode { */ buildHighlightIterator () { + if (!this.rootLanguageLayer) return new NullHighlightIterator() const layerIterators = [ this.rootLanguageLayer.buildHighlightIterator(), ...this.injectionsMarkerLayer.getMarkers().map(m => m.languageLayer.buildHighlightIterator()) @@ -134,7 +147,15 @@ class TreeSitterLanguageMode { return this.grammar.commentStrings } - isRowCommented () { + isRowCommented (row) { + const firstNonWhitespaceRange = this.buffer.findInRangeSync( + /\S/, + new Range(new Point(row, 0), new Point(row, Infinity)) + ) + if (firstNonWhitespaceRange) { + const firstNode = this.getSyntaxNodeContainingRange(firstNonWhitespaceRange) + if (firstNode) return firstNode.type.includes('comment') + } return false } @@ -265,7 +286,9 @@ class TreeSitterLanguageMode { } _forEachTreeWithRange (range, callback) { - callback(this.rootLanguageLayer.tree, this.rootLanguageLayer.grammar) + if (this.rootLanguageLayer.tree) { + callback(this.rootLanguageLayer.tree, this.rootLanguageLayer.grammar) + } const injectionMarkers = this.injectionsMarkerLayer.findMarkers({ intersectsRange: range @@ -523,11 +546,12 @@ class LanguageLayer { this.editedRange = null this.patchSinceCurrentParseStarted = new Patch() - const tree = await this.languageMode.parse( + let tree = this.languageMode.parse( this.grammar.languageModule, this.tree, includedRanges ) + if (tree.then) tree = await tree tree.buffer = this.languageMode.buffer const changes = this.patchSinceCurrentParseStarted.getChanges() @@ -1061,11 +1085,13 @@ function hasMatchingFoldSpec (specs, node) { 'increaseIndentRegexForScopeDescriptor', 'decreaseIndentRegexForScopeDescriptor', 'decreaseNextIndentRegexForScopeDescriptor', - 'regexForPattern' + 'regexForPattern', + 'getNonWordCharacters' ].forEach(methodName => { TreeSitterLanguageMode.prototype[methodName] = TextMateLanguageMode.prototype[methodName] }) TreeSitterLanguageMode.LanguageLayer = LanguageLayer +TreeSitterLanguageMode.prototype.syncOperationLimit = 1000 module.exports = TreeSitterLanguageMode