diff --git a/package.json b/package.json index e007bc635..06c73c488 100644 --- a/package.json +++ b/package.json @@ -72,7 +72,7 @@ "sinon": "1.17.4", "temp": "^0.8.3", "text-buffer": "13.14.3", - "tree-sitter": "0.12.1-0", + "tree-sitter": "0.12.1-1", "typescript-simple": "1.0.0", "underscore-plus": "^1.6.6", "winreg": "^1.2.1", diff --git a/spec/tree-sitter-language-mode-spec.js b/spec/tree-sitter-language-mode-spec.js index a788fac47..43e87d886 100644 --- a/spec/tree-sitter-language-mode-spec.js +++ b/spec/tree-sitter-language-mode-spec.js @@ -20,7 +20,7 @@ describe('TreeSitterLanguageMode', () => { }) describe('highlighting', () => { - it('applies the most specific scope mapping to each node in the syntax tree', () => { + it('applies the most specific scope mapping to each node in the syntax tree', async () => { const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, { parser: 'tree-sitter-javascript', scopes: { @@ -31,8 +31,11 @@ describe('TreeSitterLanguageMode', () => { } }) - buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar})) + const languageMode = new TreeSitterLanguageMode({buffer, grammar}) + buffer.setLanguageMode(languageMode) buffer.setText('aa.bbb = cc(d.eee());') + await languageMode.reparsePromise + expectTokensToEqual(editor, [[ {text: 'aa.', scopes: ['source']}, {text: 'bbb', scopes: ['source', 'property']}, @@ -44,7 +47,7 @@ describe('TreeSitterLanguageMode', () => { ]]) }) - it('can start or end multiple scopes at the same position', () => { + it('can start or end multiple scopes at the same position', async () => { const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, { parser: 'tree-sitter-javascript', scopes: { @@ -57,8 +60,11 @@ describe('TreeSitterLanguageMode', () => { } }) - buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar})) + const languageMode = new TreeSitterLanguageMode({buffer, grammar}) + buffer.setLanguageMode(languageMode) buffer.setText('a = bb.ccc();') + await languageMode.reparsePromise + expectTokensToEqual(editor, [[ {text: 'a', scopes: ['source', 'variable']}, {text: ' = ', scopes: ['source']}, @@ -70,7 +76,7 @@ describe('TreeSitterLanguageMode', () => { ]]) }) - it('can resume highlighting on a line that starts with whitespace', () => { + it('can resume highlighting on a line that starts with whitespace', async () => { const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, { parser: 'tree-sitter-javascript', scopes: { @@ -80,8 +86,11 @@ describe('TreeSitterLanguageMode', () => { } }) - buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar})) + const languageMode = new TreeSitterLanguageMode({buffer, grammar}) + buffer.setLanguageMode(languageMode) buffer.setText('a\n .b();') + await languageMode.reparsePromise + expectTokensToEqual(editor, [ [ {text: 'a', scopes: ['variable']}, @@ -95,7 +104,7 @@ describe('TreeSitterLanguageMode', () => { ]) }) - it('correctly skips over tokens with zero size', () => { + it('correctly skips over tokens with zero size', async () => { const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, { parser: 'tree-sitter-c', scopes: { @@ -107,10 +116,11 @@ describe('TreeSitterLanguageMode', () => { const languageMode = new TreeSitterLanguageMode({buffer, grammar}) buffer.setLanguageMode(languageMode) buffer.setText('int main() {\n int a\n int b;\n}'); + await languageMode.reparsePromise editor.screenLineForScreenRow(0) expect( - languageMode.document.rootNode.descendantForPosition(Point(1, 2), Point(1, 6)).toString() + languageMode.tree.rootNode.descendantForPosition(Point(1, 2), Point(1, 6)).toString() ).toBe('(declaration (primitive_type) (identifier) (MISSING))') expectTokensToEqual(editor, [ @@ -139,7 +149,7 @@ describe('TreeSitterLanguageMode', () => { ]) }) - it('updates lines\' highlighting when they are affected by distant changes', () => { + it('updates lines\' highlighting when they are affected by distant changes', async () => { const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, { parser: 'tree-sitter-javascript', scopes: { @@ -148,10 +158,12 @@ describe('TreeSitterLanguageMode', () => { } }) - buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar})) + const languageMode = new TreeSitterLanguageMode({buffer, grammar}) + buffer.setLanguageMode(languageMode) + buffer.setText('a(\nb,\nc\n') + await languageMode.reparsePromise // missing closing paren - buffer.setText('a(\nb,\nc\n') expectTokensToEqual(editor, [ [{text: 'a(', scopes: []}], [{text: 'b,', scopes: []}], @@ -160,6 +172,7 @@ describe('TreeSitterLanguageMode', () => { ]) buffer.append(')') + await languageMode.reparsePromise expectTokensToEqual(editor, [ [ {text: 'a', scopes: ['function']}, @@ -171,7 +184,7 @@ describe('TreeSitterLanguageMode', () => { ]) }) - it('handles edits after tokens that end between CR and LF characters (regression)', () => { + it('handles edits after tokens that end between CR and LF characters (regression)', async () => { const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, { parser: 'tree-sitter-javascript', scopes: { @@ -181,13 +194,14 @@ describe('TreeSitterLanguageMode', () => { } }) - buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar})) - + const languageMode = new TreeSitterLanguageMode({buffer, grammar}) + buffer.setLanguageMode(languageMode) buffer.setText([ '// abc', '', 'a("b").c' ].join('\r\n')) + await languageMode.reparsePromise expectTokensToEqual(editor, [ [{text: '// abc', scopes: ['comment']}], @@ -201,6 +215,7 @@ describe('TreeSitterLanguageMode', () => { ]) buffer.insert([2, 0], ' ') + await languageMode.reparsePromise expectTokensToEqual(editor, [ [{text: '// abc', scopes: ['comment']}], [{text: '', scopes: []}], @@ -220,7 +235,7 @@ describe('TreeSitterLanguageMode', () => { editor.displayLayer.reset({foldCharacter: '…'}) }) - it('can fold nodes that start and end with specified tokens', () => { + it('can fold nodes that start and end with specified tokens', async () => { const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, { parser: 'tree-sitter-javascript', folds: [ @@ -235,7 +250,8 @@ describe('TreeSitterLanguageMode', () => { ] }) - buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar})) + const languageMode = new TreeSitterLanguageMode({buffer, grammar}) + buffer.setLanguageMode(languageMode) buffer.setText(dedent ` module.exports = class A { @@ -246,6 +262,7 @@ describe('TreeSitterLanguageMode', () => { } } `) + await languageMode.reparsePromise editor.screenLineForScreenRow(0) @@ -275,7 +292,7 @@ describe('TreeSitterLanguageMode', () => { `) }) - it('can fold nodes of specified types', () => { + it('can fold nodes of specified types', async () => { const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, { parser: 'tree-sitter-javascript', folds: [ @@ -296,7 +313,8 @@ describe('TreeSitterLanguageMode', () => { ] }) - buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar})) + const languageMode = new TreeSitterLanguageMode({buffer, grammar}) + buffer.setLanguageMode(languageMode) buffer.setText(dedent ` const element1 = { world `) + await languageMode.reparsePromise editor.screenLineForScreenRow(0) @@ -336,7 +355,7 @@ describe('TreeSitterLanguageMode', () => { `) }) - it('can fold entire nodes when no start or end parameters are specified', () => { + it('can fold entire nodes when no start or end parameters are specified', async () => { const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, { parser: 'tree-sitter-javascript', folds: [ @@ -346,7 +365,8 @@ describe('TreeSitterLanguageMode', () => { ] }) - buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar})) + const languageMode = new TreeSitterLanguageMode({buffer, grammar}) + buffer.setLanguageMode(languageMode) buffer.setText(dedent ` /** * Important @@ -355,6 +375,7 @@ describe('TreeSitterLanguageMode', () => { Also important */ `) + await languageMode.reparsePromise editor.screenLineForScreenRow(0) @@ -379,7 +400,7 @@ describe('TreeSitterLanguageMode', () => { `) }) - it('tries each folding strategy for a given node in the order specified', () => { + it('tries each folding strategy for a given node in the order specified', async () => { const grammar = new TreeSitterGrammar(atom.grammars, cGrammarPath, { parser: 'tree-sitter-c', folds: [ @@ -405,8 +426,8 @@ describe('TreeSitterLanguageMode', () => { ] }) - buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar})) - + const languageMode = new TreeSitterLanguageMode({buffer, grammar}) + buffer.setLanguageMode(languageMode) buffer.setText(dedent ` #ifndef FOO_H_ #define FOO_H_ @@ -430,6 +451,7 @@ describe('TreeSitterLanguageMode', () => { #endif `) + await languageMode.reparsePromise editor.screenLineForScreenRow(0) @@ -504,8 +526,6 @@ describe('TreeSitterLanguageMode', () => { ] }) - buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar})) - buffer.setText(dedent ` def ab(): print 'a' @@ -515,6 +535,7 @@ describe('TreeSitterLanguageMode', () => { print 'c' print 'd' `) + buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar})) editor.screenLineForScreenRow(0) @@ -537,9 +558,8 @@ describe('TreeSitterLanguageMode', () => { parser: 'tree-sitter-javascript' }) - buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar})) - buffer.setText('foo({bar: baz});') + buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar})) editor.screenLineForScreenRow(0) expect(editor.scopeDescriptorForBufferPosition([0, 6]).getScopesArray()).toEqual([ @@ -562,13 +582,13 @@ describe('TreeSitterLanguageMode', () => { scopes: {'program': 'source'} }) - buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar})) buffer.setText(dedent ` function a (b, c, d) { eee.f() g() } `) + buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar})) editor.screenLineForScreenRow(0) diff --git a/src/tree-sitter-language-mode.js b/src/tree-sitter-language-mode.js index 0d2fab8cf..3ec6a037a 100644 --- a/src/tree-sitter-language-mode.js +++ b/src/tree-sitter-language-mode.js @@ -1,4 +1,4 @@ -const {Document} = require('tree-sitter') +const Parser = require('tree-sitter') const {Point, Range} = require('text-buffer') const {Emitter, Disposable} = require('event-kit') const ScopeDescriptor = require('./scope-descriptor') @@ -14,13 +14,20 @@ class TreeSitterLanguageMode { this.buffer = buffer this.grammar = grammar this.config = config - this.document = new Document() - this.document.setInput(new TreeSitterTextBufferInput(buffer)) - this.document.setLanguage(grammar.languageModule) - this.document.parse() + this.parser = new Parser() + this.parser.setLanguage(grammar.languageModule) + this.tree = this.parser.parseTextBufferSync(this.buffer.buffer) this.rootScopeDescriptor = new ScopeDescriptor({scopes: [this.grammar.id]}) this.emitter = new Emitter() this.isFoldableCache = [] + this.hasQueuedParse = false + this.buffer.onDidChangeText(async () => { + if (!this.reparsePromise) { + this.reparsePromise = this.reparse().then(() => { + this.reparsePromise = null + }) + } + }) // TODO: Remove this once TreeSitterLanguageMode implements its own auto-indentation system. This // is temporarily needed in order to delegate to the TextMateLanguageMode's auto-indent system. @@ -36,7 +43,7 @@ class TreeSitterLanguageMode { const oldEndRow = oldRange.end.row const newEndRow = newRange.end.row this.isFoldableCache.splice(startRow, oldEndRow - startRow, ...new Array(newEndRow - startRow)) - this.document.edit({ + this.tree.edit({ startIndex: this.buffer.characterIndexForPosition(oldRange.start), lengthRemoved: oldText.length, lengthAdded: newText.length, @@ -50,8 +57,10 @@ class TreeSitterLanguageMode { Section - Highlighting */ - buildHighlightIterator () { - const invalidatedRanges = this.document.parse() + async reparse () { + const tree = await this.parser.parseTextBuffer(this.buffer.buffer, this.tree) + const invalidatedRanges = tree.getChangedRanges(this.tree) + this.tree = tree for (let i = 0, n = invalidatedRanges.length; i < n; i++) { const range = invalidatedRanges[i] const startRow = range.start.row @@ -61,6 +70,9 @@ class TreeSitterLanguageMode { } this.emitter.emit('did-change-highlighting', range) } + } + + buildHighlightIterator () { return new TreeSitterHighlightIterator(this) } @@ -139,7 +151,7 @@ class TreeSitterLanguageMode { getFoldableRangesAtIndentLevel (goalLevel) { let result = [] - let stack = [{node: this.document.rootNode, level: 0}] + let stack = [{node: this.tree.rootNode, level: 0}] while (stack.length > 0) { const {node, level} = stack.pop() @@ -183,7 +195,7 @@ class TreeSitterLanguageMode { } getFoldableRangeContainingPoint (point, tabLength, existenceOnly = false) { - let node = this.document.rootNode.descendantForPosition(this.buffer.clipPosition(point)) + let node = this.tree.rootNode.descendantForPosition(this.buffer.clipPosition(point)) while (node) { if (existenceOnly && node.startPosition.row < point.row) break if (node.endPosition.row > point.row) { @@ -273,7 +285,7 @@ class TreeSitterLanguageMode { getRangeForSyntaxNodeContainingRange (range) { const startIndex = this.buffer.characterIndexForPosition(range.start) const endIndex = this.buffer.characterIndexForPosition(range.end) - let node = this.document.rootNode.descendantForIndex(startIndex, endIndex - 1) + let node = this.tree.rootNode.descendantForIndex(startIndex, endIndex - 1) while (node && node.startIndex === startIndex && node.endIndex === endIndex) { node = node.parent } @@ -305,7 +317,7 @@ class TreeSitterLanguageMode { scopeDescriptorForPosition (point) { point = Point.fromObject(point) const result = [] - let node = this.document.rootNode.descendantForPosition(point) + let node = this.tree.rootNode.descendantForPosition(point) // Don't include anonymous token types like '(' because they prevent scope chains // from being parsed as CSS selectors by the `slick` parser. Other css selector @@ -331,17 +343,17 @@ class TreeSitterLanguageMode { } class TreeSitterHighlightIterator { - constructor (layer, document) { + constructor (layer) { this.layer = layer + this.treeCursor = this.layer.tree.walk() // Conceptually, the iterator represents a single position in the text. It stores this // position both as a character index and as a `Point`. This position corresponds to a // leaf node of the syntax tree, which either contains or follows the iterator's - // textual position. The `currentNode` property represents that leaf node, and + // textual position. The `treeCursor` property points at that leaf node, and // `currentChildIndex` represents the child index of that leaf node within its parent. this.currentIndex = null this.currentPosition = null - this.currentNode = null this.currentChildIndex = null // In order to determine which selectors match its current node, the iterator maintains @@ -358,6 +370,8 @@ class TreeSitterHighlightIterator { } seek (targetPosition) { + while (this.treeCursor.gotoParent()) {} + const containingTags = [] this.closeTags.length = 0 @@ -367,33 +381,28 @@ class TreeSitterHighlightIterator { this.currentPosition = targetPosition this.currentIndex = this.layer.buffer.characterIndexForPosition(targetPosition) - var node = this.layer.document.rootNode var childIndex = -1 var nodeContainsTarget = true for (;;) { - this.currentNode = node this.currentChildIndex = childIndex if (!nodeContainsTarget) break - this.containingNodeTypes.push(node.type) + this.containingNodeTypes.push(this.treeCursor.nodeType) this.containingNodeChildIndices.push(childIndex) const scopeName = this.currentScopeName() if (scopeName) { const id = this.layer.grammar.idForScope(scopeName) - if (this.currentIndex === node.startIndex) { + if (this.currentIndex === this.treeCursor.startIndex) { this.openTags.push(id) } else { containingTags.push(id) } } - node = node.firstChildForIndex(this.currentIndex) - if (node) { - if (node.startIndex > this.currentIndex) nodeContainsTarget = false - childIndex = node.childIndex - } else { - break - } + const nextChildIndex = this.treeCursor.gotoFirstChildForIndex(this.currentIndex) + if (nextChildIndex == null) break + if (this.treeCursor.startIndex > this.currentIndex) nodeContainsTarget = false + childIndex = nextChildIndex } return containingTags @@ -403,42 +412,35 @@ class TreeSitterHighlightIterator { this.closeTags.length = 0 this.openTags.length = 0 - if (!this.currentNode) { - this.currentPosition = {row: Infinity, column: Infinity} - return false - } - do { - if (this.currentIndex < this.currentNode.startIndex) { - this.currentIndex = this.currentNode.startIndex - this.currentPosition = this.currentNode.startPosition + if (this.currentIndex < this.treeCursor.startIndex) { + this.currentIndex = this.treeCursor.startIndex + this.currentPosition = this.treeCursor.startPosition this.pushOpenTag() this.descendLeft() - } else if (this.currentIndex < this.currentNode.endIndex) { + } else if (this.currentIndex < this.treeCursor.endIndex) { while (true) { - this.currentIndex = this.currentNode.endIndex - this.currentPosition = this.currentNode.endPosition + this.currentIndex = this.treeCursor.endIndex + this.currentPosition = this.treeCursor.endPosition this.pushCloseTag() - const {nextSibling} = this.currentNode - if (nextSibling && nextSibling.endIndex > this.currentIndex) { - this.currentNode = nextSibling + if (this.treeCursor.gotoNextSibling()) { this.currentChildIndex++ - if (this.currentIndex === nextSibling.startIndex) { + if (this.currentIndex === this.treeCursor.startIndex) { this.pushOpenTag() this.descendLeft() } break } else { - this.currentNode = this.currentNode.parent this.currentChildIndex = last(this.containingNodeChildIndices) - if (!this.currentNode) break + if (!this.treeCursor.gotoParent()) break } } - } else { - this.currentNode = this.currentNode.nextSibling + } else if (!this.treeCursor.gotoNextSibling()) { + this.currentPosition = {row: Infinity, column: Infinity} + break } - } while (this.closeTags.length === 0 && this.openTags.length === 0 && this.currentNode) + } while (this.closeTags.length === 0 && this.openTags.length === 0) return true } @@ -458,9 +460,7 @@ class TreeSitterHighlightIterator { // Private methods descendLeft () { - let child - while ((child = this.currentNode.firstChild) && this.currentIndex === child.startIndex) { - this.currentNode = child + while (this.treeCursor.gotoFirstChild()) { this.currentChildIndex = 0 this.pushOpenTag() } @@ -470,7 +470,7 @@ class TreeSitterHighlightIterator { return this.layer.grammar.scopeMap.get( this.containingNodeTypes, this.containingNodeChildIndices, - this.currentNode.isNamed + this.treeCursor.nodeIsNamed ) } @@ -482,37 +482,13 @@ class TreeSitterHighlightIterator { } pushOpenTag () { - this.containingNodeTypes.push(this.currentNode.type) + this.containingNodeTypes.push(this.treeCursor.nodeType) this.containingNodeChildIndices.push(this.currentChildIndex) const scopeName = this.currentScopeName() if (scopeName) this.openTags.push(this.layer.grammar.idForScope(scopeName)) } } -class TreeSitterTextBufferInput { - constructor (buffer) { - this.buffer = buffer - this.position = {row: 0, column: 0} - this.isBetweenCRLF = false - } - - seek (offset, position) { - this.position = position - this.isBetweenCRLF = this.position.column > this.buffer.lineLengthForRow(this.position.row) - } - - read () { - const endPosition = this.buffer.clipPosition(new Point(this.position.row + 1000, 0)) - let text = this.buffer.getTextInRange([this.position, endPosition]) - if (this.isBetweenCRLF) { - text = text.slice(1) - this.isBetweenCRLF = false - } - this.position = endPosition - return text - } -} - function last (array) { return array[array.length - 1] }