github/atom
1
1
Fork 0
mirror of https://github.com/atom/atom.git synced 2026-01-24 22:38:20 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #17339 from atom/mb-async-parsing

Browse Source 
Perform parsing off the main thread when Tree-sitter is enabled
This commit is contained in:
Max Brunsfeld
2018-05-23 16:45:51 -07:00
committed by GitHub
parent 6a5acd95f7 53dfa83ae9
commit a78d682e00
4 changed files with 278 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
package.json
View File

@@ -71,8 +71,8 @@
"service-hub": "^0.7.4",
"sinon": "1.17.4",
"temp": "^0.8.3",
"text-buffer": "13.14.2",
"tree-sitter": "^0.11.2",
"text-buffer": "13.14.3",
"tree-sitter": "0.12.6",
"typescript-simple": "1.0.0",
"underscore-plus": "^1.6.6",
"winreg": "^1.2.1",

137
spec/tree-sitter-language-mode-spec.js
View File

@@ -20,7 +20,7 @@ describe('TreeSitterLanguageMode', () => {
})
describe('highlighting', () => {
it('applies the most specific scope mapping to each node in the syntax tree', () => {
it('applies the most specific scope mapping to each node in the syntax tree', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
scopes: {
@@ -31,8 +31,11 @@ describe('TreeSitterLanguageMode', () => {
}
})
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText('aa.bbb = cc(d.eee());')
await languageMode.reparsePromise
expectTokensToEqual(editor, [[
{text: 'aa.', scopes: ['source']},
{text: 'bbb', scopes: ['source', 'property']},
@@ -44,7 +47,7 @@ describe('TreeSitterLanguageMode', () => {
]])
})
it('can start or end multiple scopes at the same position', () => {
it('can start or end multiple scopes at the same position', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
scopes: {
@@ -57,8 +60,11 @@ describe('TreeSitterLanguageMode', () => {
}
})
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText('a = bb.ccc();')
await languageMode.reparsePromise
expectTokensToEqual(editor, [[
{text: 'a', scopes: ['source', 'variable']},
{text: ' = ', scopes: ['source']},
@@ -70,7 +76,7 @@ describe('TreeSitterLanguageMode', () => {
]])
})
it('can resume highlighting on a line that starts with whitespace', () => {
it('can resume highlighting on a line that starts with whitespace', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
scopes: {
@@ -80,8 +86,11 @@ describe('TreeSitterLanguageMode', () => {
}
})
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText('a\n .b();')
await languageMode.reparsePromise
expectTokensToEqual(editor, [
[
{text: 'a', scopes: ['variable']},
@@ -95,7 +104,7 @@ describe('TreeSitterLanguageMode', () => {
])
})
it('correctly skips over tokens with zero size', () => {
it('correctly skips over tokens with zero size', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-c',
scopes: {
@@ -107,10 +116,11 @@ describe('TreeSitterLanguageMode', () => {
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText('int main() {\n int a\n int b;\n}');
await languageMode.reparsePromise
editor.screenLineForScreenRow(0)
expect(
languageMode.document.rootNode.descendantForPosition(Point(1, 2), Point(1, 6)).toString()
languageMode.tree.rootNode.descendantForPosition(Point(1, 2), Point(1, 6)).toString()
).toBe('(declaration (primitive_type) (identifier) (MISSING))')
expectTokensToEqual(editor, [
@@ -139,7 +149,7 @@ describe('TreeSitterLanguageMode', () => {
])
})
it('updates lines\' highlighting when they are affected by distant changes', () => {
it('updates lines\' highlighting when they are affected by distant changes', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
scopes: {
@@ -148,10 +158,12 @@ describe('TreeSitterLanguageMode', () => {
}
})
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText('a(\nb,\nc\n')
await languageMode.reparsePromise
// missing closing paren
buffer.setText('a(\nb,\nc\n')
expectTokensToEqual(editor, [
[{text: 'a(', scopes: []}],
[{text: 'b,', scopes: []}],
@@ -160,6 +172,7 @@ describe('TreeSitterLanguageMode', () => {
])
buffer.append(')')
await languageMode.reparsePromise
expectTokensToEqual(editor, [
[
{text: 'a', scopes: ['function']},
@@ -171,7 +184,7 @@ describe('TreeSitterLanguageMode', () => {
])
})
it('handles edits after tokens that end between CR and LF characters (regression)', () => {
it('handles edits after tokens that end between CR and LF characters (regression)', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
scopes: {
@@ -181,13 +194,14 @@ describe('TreeSitterLanguageMode', () => {
}
})
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText([
'// abc',
'',
'a("b").c'
].join('\r\n'))
await languageMode.reparsePromise
expectTokensToEqual(editor, [
[{text: '// abc', scopes: ['comment']}],
@@ -201,6 +215,7 @@ describe('TreeSitterLanguageMode', () => {
])
buffer.insert([2, 0], ' ')
await languageMode.reparsePromise
expectTokensToEqual(editor, [
[{text: '// abc', scopes: ['comment']}],
[{text: '', scopes: []}],
@@ -213,6 +228,67 @@ describe('TreeSitterLanguageMode', () => {
]
])
})
describe('when the buffer changes during a parse', () => {
it('immediately parses again when the current parse completes', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
scopes: {
'identifier': 'variable',
'call_expression > identifier': 'function',
'new_expression > call_expression > identifier': 'constructor'
}
})
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText('abc;');
await languageMode.reparsePromise
expectTokensToEqual(editor, [
[
{text: 'abc', scopes: ['variable']},
{text: ';', scopes: []}
],
])
buffer.setTextInRange([[0, 3], [0, 3]], '()');
expectTokensToEqual(editor, [
[
{text: 'abc()', scopes: ['variable']},
{text: ';', scopes: []}
],
])
buffer.setTextInRange([[0, 0], [0, 0]], 'new ');
expectTokensToEqual(editor, [
[
{text: 'new ', scopes: []},
{text: 'abc()', scopes: ['variable']},
{text: ';', scopes: []}
],
])
await languageMode.reparsePromise
expect(languageMode.reparsePromise).not.toBeNull()
expectTokensToEqual(editor, [
[
{text: 'new ', scopes: []},
{text: 'abc', scopes: ['function']},
{text: '();', scopes: []}
],
])
await languageMode.reparsePromise
expect(languageMode.reparsePromise).toBeNull()
expectTokensToEqual(editor, [
[
{text: 'new ', scopes: []},
{text: 'abc', scopes: ['constructor']},
{text: '();', scopes: []}
],
])
})
})
})
describe('folding', () => {
@@ -220,7 +296,7 @@ describe('TreeSitterLanguageMode', () => {
editor.displayLayer.reset({foldCharacter: '…'})
})
it('can fold nodes that start and end with specified tokens', () => {
it('can fold nodes that start and end with specified tokens', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
folds: [
@@ -235,7 +311,8 @@ describe('TreeSitterLanguageMode', () => {
]
})
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText(dedent `
module.exports =
class A {
@@ -246,6 +323,7 @@ describe('TreeSitterLanguageMode', () => {
}
}
`)
await languageMode.reparsePromise
editor.screenLineForScreenRow(0)
@@ -275,7 +353,7 @@ describe('TreeSitterLanguageMode', () => {
`)
})
it('can fold nodes of specified types', () => {
it('can fold nodes of specified types', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
folds: [
@@ -296,7 +374,8 @@ describe('TreeSitterLanguageMode', () => {
]
})
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText(dedent `
const element1 = <Element
className='submit'
@@ -307,6 +386,7 @@ describe('TreeSitterLanguageMode', () => {
<span>world</span>
</Element>
`)
await languageMode.reparsePromise
editor.screenLineForScreenRow(0)
@@ -336,7 +416,7 @@ describe('TreeSitterLanguageMode', () => {
`)
})
it('can fold entire nodes when no start or end parameters are specified', () => {
it('can fold entire nodes when no start or end parameters are specified', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
folds: [
@@ -346,7 +426,8 @@ describe('TreeSitterLanguageMode', () => {
]
})
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText(dedent `
/**
* Important
@@ -355,6 +436,7 @@ describe('TreeSitterLanguageMode', () => {
Also important
*/
`)
await languageMode.reparsePromise
editor.screenLineForScreenRow(0)
@@ -379,7 +461,7 @@ describe('TreeSitterLanguageMode', () => {
`)
})
it('tries each folding strategy for a given node in the order specified', () => {
it('tries each folding strategy for a given node in the order specified', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, cGrammarPath, {
parser: 'tree-sitter-c',
folds: [
@@ -405,8 +487,8 @@ describe('TreeSitterLanguageMode', () => {
]
})
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText(dedent `
#ifndef FOO_H_
#define FOO_H_
@@ -430,6 +512,7 @@ describe('TreeSitterLanguageMode', () => {
#endif
`)
await languageMode.reparsePromise
editor.screenLineForScreenRow(0)
@@ -504,8 +587,6 @@ describe('TreeSitterLanguageMode', () => {
]
})
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
buffer.setText(dedent `
def ab():
print 'a'
@@ -515,6 +596,7 @@ describe('TreeSitterLanguageMode', () => {
print 'c'
print 'd'
`)
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
editor.screenLineForScreenRow(0)
@@ -537,9 +619,8 @@ describe('TreeSitterLanguageMode', () => {
parser: 'tree-sitter-javascript'
})
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
buffer.setText('foo({bar: baz});')
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
editor.screenLineForScreenRow(0)
expect(editor.scopeDescriptorForBufferPosition([0, 6]).getScopesArray()).toEqual([
@@ -562,13 +643,13 @@ describe('TreeSitterLanguageMode', () => {
scopes: {'program': 'source'}
})
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
buffer.setText(dedent `
function a (b, c, d) {
eee.f()
g()
}
`)
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
editor.screenLineForScreenRow(0)

34
src/project.js
View File

@@ -662,27 +662,35 @@ class Project extends Model {
// * `text` The {String} text to use as a buffer.
//
// Returns a {Promise} that resolves to the {TextBuffer}.
buildBuffer (absoluteFilePath) {
async buildBuffer (absoluteFilePath) {
const params = {shouldDestroyOnFileDelete: this.shouldDestroyBufferOnFileDelete}
let promise
let buffer
if (absoluteFilePath != null) {
if (this.loadPromisesByPath[absoluteFilePath] == null) {
this.loadPromisesByPath[absoluteFilePath] =
TextBuffer.load(absoluteFilePath, params).catch(error => {
delete this.loadPromisesByPath[absoluteFilePath]
throw error
})
TextBuffer.load(absoluteFilePath, params)
.then(result => {
delete this.loadPromisesByPath[absoluteFilePath]
return result
})
.catch(error => {
delete this.loadPromisesByPath[absoluteFilePath]
throw error
})
}
promise = this.loadPromisesByPath[absoluteFilePath]
buffer = await this.loadPromisesByPath[absoluteFilePath]
} else {
promise = Promise.resolve(new TextBuffer(params))
buffer = new TextBuffer(params)
}
return promise.then(buffer => {
delete this.loadPromisesByPath[absoluteFilePath]
this.addBuffer(buffer)
return buffer
})
this.grammarRegistry.autoAssignLanguageMode(buffer)
if (buffer.languageMode.initialize) {
await buffer.languageMode.initialize()
}
this.addBuffer(buffer)
return buffer
}
addBuffer (buffer, options = {}) {

250
src/tree-sitter-language-mode.js
View File

@@ -1,4 +1,4 @@
const {Document} = require('tree-sitter')
const Parser = require('tree-sitter')
const {Point, Range} = require('text-buffer')
const {Emitter, Disposable} = require('event-kit')
const ScopeDescriptor = require('./scope-descriptor')
@@ -14,44 +14,73 @@ class TreeSitterLanguageMode {
this.buffer = buffer
this.grammar = grammar
this.config = config
this.document = new Document()
this.document.setInput(new TreeSitterTextBufferInput(buffer))
this.document.setLanguage(grammar.languageModule)
this.document.parse()
this.parser = new Parser()
this.parser.setLanguage(grammar.languageModule)
this.tree = null
this.rootScopeDescriptor = new ScopeDescriptor({scopes: [this.grammar.id]})
this.emitter = new Emitter()
this.isFoldableCache = []
this.hasQueuedParse = false
this.changeListsSinceCurrentParse = []
this.buffer.onDidChangeText(async ({changes}) => {
if (this.reparsePromise) {
this.changeListsSinceCurrentParse.push(changes)
} else {
this.reparsePromise = this.reparse()
}
})
// TODO: Remove this once TreeSitterLanguageMode implements its own auto-indentation system. This
// is temporarily needed in order to delegate to the TextMateLanguageMode's auto-indent system.
this.regexesByPattern = {}
}
async initialize () {
this.tree = await this.parser.parseTextBuffer(this.buffer.buffer)
}
ensureParseTree () {
if (!this.tree) {
this.tree = this.parser.parseTextBufferSync(this.buffer.buffer)
}
}
getLanguageId () {
return this.grammar.id
}
bufferDidChange ({oldRange, newRange, oldText, newText}) {
bufferDidChange (change) {
this.ensureParseTree()
const {oldRange, newRange} = change
const startRow = oldRange.start.row
const oldEndRow = oldRange.end.row
const newEndRow = newRange.end.row
this.isFoldableCache.splice(startRow, oldEndRow - startRow, ...new Array(newEndRow - startRow))
this.document.edit({
startIndex: this.buffer.characterIndexForPosition(oldRange.start),
lengthRemoved: oldText.length,
lengthAdded: newText.length,
startPosition: oldRange.start,
extentRemoved: oldRange.getExtent(),
extentAdded: newRange.getExtent()
})
this.tree.edit(this.treeEditForBufferChange(change))
}
/*
Section - Highlighting
*/
buildHighlightIterator () {
const invalidatedRanges = this.document.parse()
treeEditForBufferChange ({oldRange, newRange, oldText, newText}) {
const startIndex = this.buffer.characterIndexForPosition(oldRange.start)
return {
startIndex,
oldEndIndex: startIndex + oldText.length,
newEndIndex: startIndex + newText.length,
startPosition: oldRange.start,
oldEndPosition: oldRange.end,
newEndPosition: newRange.end
}
}
async reparse () {
const tree = await this.parser.parseTextBuffer(this.buffer.buffer, this.tree, {
syncOperationLimit: 1000
})
const invalidatedRanges = this.tree.getChangedRanges(tree)
for (let i = 0, n = invalidatedRanges.length; i < n; i++) {
const range = invalidatedRanges[i]
const startRow = range.start.row
@@ -61,7 +90,24 @@ class TreeSitterLanguageMode {
}
this.emitter.emit('did-change-highlighting', range)
}
return new TreeSitterHighlightIterator(this)
this.tree = tree
if (this.changeListsSinceCurrentParse.length > 0) {
for (const changeList of this.changeListsSinceCurrentParse) {
for (let i = changeList.length - 1; i >= 0; i--) {
this.tree.edit(this.treeEditForBufferChange(changeList[i]))
}
}
this.changeListsSinceCurrentParse.length = 0
this.reparsePromise = this.reparse()
} else {
this.reparsePromise = null
}
}
buildHighlightIterator () {
this.ensureParseTree()
return new TreeSitterHighlightIterator(this, this.tree.walk())
}
onDidChangeHighlighting (callback) {
@@ -138,8 +184,9 @@ class TreeSitterLanguageMode {
}
getFoldableRangesAtIndentLevel (goalLevel) {
this.ensureParseTree()
let result = []
let stack = [{node: this.document.rootNode, level: 0}]
let stack = [{node: this.tree.rootNode, level: 0}]
while (stack.length > 0) {
const {node, level} = stack.pop()
@@ -183,7 +230,8 @@ class TreeSitterLanguageMode {
}
getFoldableRangeContainingPoint (point, tabLength, existenceOnly = false) {
let node = this.document.rootNode.descendantForPosition(this.buffer.clipPosition(point))
this.ensureParseTree()
let node = this.tree.rootNode.descendantForPosition(this.buffer.clipPosition(point))
while (node) {
if (existenceOnly && node.startPosition.row < point.row) break
if (node.endPosition.row > point.row) {
@@ -273,7 +321,7 @@ class TreeSitterLanguageMode {
getRangeForSyntaxNodeContainingRange (range) {
const startIndex = this.buffer.characterIndexForPosition(range.start)
const endIndex = this.buffer.characterIndexForPosition(range.end)
let node = this.document.rootNode.descendantForIndex(startIndex, endIndex - 1)
let node = this.tree.rootNode.descendantForIndex(startIndex, endIndex - 1)
while (node && node.startIndex === startIndex && node.endIndex === endIndex) {
node = node.parent
}
@@ -303,9 +351,9 @@ class TreeSitterLanguageMode {
}
scopeDescriptorForPosition (point) {
this.ensureParseTree()
point = Point.fromObject(point)
const result = []
let node = this.document.rootNode.descendantForPosition(point)
let node = this.tree.rootNode.descendantForPosition(point)
// Don't include anonymous token types like '(' because they prevent scope chains
// from being parsed as CSS selectors by the `slick` parser. Other css selector
@@ -313,6 +361,7 @@ class TreeSitterLanguageMode {
// selectors.
if (!node.isNamed) node = node.parent
const result = []
while (node) {
result.push(node.type)
node = node.parent
@@ -331,18 +380,9 @@ class TreeSitterLanguageMode {
}
class TreeSitterHighlightIterator {
constructor (layer, document) {
this.layer = layer
// Conceptually, the iterator represents a single position in the text. It stores this
// position both as a character index and as a `Point`. This position corresponds to a
// leaf node of the syntax tree, which either contains or follows the iterator's
// textual position. The `currentNode` property represents that leaf node, and
// `currentChildIndex` represents the child index of that leaf node within its parent.
this.currentIndex = null
this.currentPosition = null
this.currentNode = null
this.currentChildIndex = null
constructor (languageMode, treeCursor) {
this.languageMode = languageMode
this.treeCursor = treeCursor
// In order to determine which selectors match its current node, the iterator maintains
// a list of the current node's ancestors. Because the selectors can use the `:nth-child`
@@ -350,6 +390,15 @@ class TreeSitterHighlightIterator {
this.containingNodeTypes = []
this.containingNodeChildIndices = []
// Conceptually, the iterator represents a single position in the text. It stores this
// position both as a character index and as a `Point`. This position corresponds to a
// leaf node of the syntax tree, which either contains or follows the iterator's
// textual position. The `treeCursor` property points at that leaf node, and
// `currentChildIndex` represents the child index of that leaf node within its parent.
this.currentIndex = null
this.currentPosition = null
this.currentChildIndex = null
// At any given position, the iterator exposes the list of class names that should be
// *ended* at its current position and the list of class names that should be *started*
// at its current position.
@@ -358,6 +407,8 @@ class TreeSitterHighlightIterator {
}
seek (targetPosition) {
while (this.treeCursor.gotoParent()) {}
const containingTags = []
this.closeTags.length = 0
@@ -365,35 +416,33 @@ class TreeSitterHighlightIterator {
this.containingNodeTypes.length = 0
this.containingNodeChildIndices.length = 0
this.currentPosition = targetPosition
this.currentIndex = this.layer.buffer.characterIndexForPosition(targetPosition)
this.currentIndex = this.languageMode.buffer.characterIndexForPosition(targetPosition)
var node = this.layer.document.rootNode
// Descend from the root of the tree to the smallest node that spans the given position.
// Keep track of any nodes along the way that are associated with syntax highlighting
// tags. These tags must be returned.
var childIndex = -1
var nodeContainsTarget = true
for (;;) {
this.currentNode = node
this.currentChildIndex = childIndex
if (!nodeContainsTarget) break
this.containingNodeTypes.push(node.type)
this.containingNodeTypes.push(this.treeCursor.nodeType)
this.containingNodeChildIndices.push(childIndex)
const scopeName = this.currentScopeName()
if (scopeName) {
const id = this.layer.grammar.idForScope(scopeName)
if (this.currentIndex === node.startIndex) {
const id = this.languageMode.grammar.idForScope(scopeName)
if (this.currentIndex === this.treeCursor.startIndex) {
this.openTags.push(id)
} else {
containingTags.push(id)
}
}
node = node.firstChildForIndex(this.currentIndex)
if (node) {
if (node.startIndex > this.currentIndex) nodeContainsTarget = false
childIndex = node.childIndex
} else {
break
}
const nextChildIndex = this.treeCursor.gotoFirstChildForIndex(this.currentIndex)
if (nextChildIndex == null) break
if (this.treeCursor.startIndex > this.currentIndex) nodeContainsTarget = false
childIndex = nextChildIndex
}
return containingTags
@@ -403,42 +452,61 @@ class TreeSitterHighlightIterator {
this.closeTags.length = 0
this.openTags.length = 0
if (!this.currentNode) {
this.currentPosition = {row: Infinity, column: Infinity}
return false
}
// Step forward through the leaves of the tree to find the next place where one or more
// syntax highlighting tags begin, end, or both.
do {
if (this.currentIndex < this.currentNode.startIndex) {
this.currentIndex = this.currentNode.startIndex
this.currentPosition = this.currentNode.startPosition
// If the iterator is before the beginning of the current node, advance it to the
// beginning of then node and then walk down into the node's children, marking
// open tags as needed.
if (this.currentIndex < this.treeCursor.startIndex) {
this.currentIndex = this.treeCursor.startIndex
this.currentPosition = this.treeCursor.startPosition
this.pushOpenTag()
this.descendLeft()
} else if (this.currentIndex < this.currentNode.endIndex) {
while (true) {
this.currentIndex = this.currentNode.endIndex
this.currentPosition = this.currentNode.endPosition
// If the iterator is within the current node, advance it to the end of the node
// and then walk up the tree until the next sibling is found, marking close tags
// as needed.
//
} else if (this.currentIndex < this.treeCursor.endIndex) {
/* eslint-disable no-labels */
ascendingLoop:
do {
this.currentIndex = this.treeCursor.endIndex
this.currentPosition = this.treeCursor.endPosition
this.pushCloseTag()
const {nextSibling} = this.currentNode
if (nextSibling && nextSibling.endIndex > this.currentIndex) {
this.currentNode = nextSibling
// Stop walking upward when we reach a node with a next sibling.
while (this.treeCursor.gotoNextSibling()) {
this.currentChildIndex++
if (this.currentIndex === nextSibling.startIndex) {
// If the next sibling has a size of zero (e.g. something like an `automatic_semicolon`,
// an `indent`, or a `MISSING` node inserted by the parser during error recovery),
// then skip it. These nodes play no role in syntax highlighting.
if (this.treeCursor.endIndex === this.currentIndex) continue
// If the next sibling starts right at the end of the current node (i.e. there is
// no whitespace in between), then before returning, also mark any open tags associated
// with this point in the tree.
if (this.treeCursor.startIndex === this.currentIndex) {
this.pushOpenTag()
this.descendLeft()
}
break
} else {
this.currentNode = this.currentNode.parent
this.currentChildIndex = last(this.containingNodeChildIndices)
if (!this.currentNode) break
break ascendingLoop
}
}
} else {
this.currentNode = this.currentNode.nextSibling
this.currentChildIndex = last(this.containingNodeChildIndices)
} while (this.treeCursor.gotoParent())
/* eslint-disable no-labels */
// If the iterator is at the end of a node, advance to the node's next sibling. If
// it has no next sibing, then the iterator has reached the end of the tree.
} else if (!this.treeCursor.gotoNextSibling()) {
this.currentPosition = {row: Infinity, column: Infinity}
break
}
} while (this.closeTags.length === 0 && this.openTags.length === 0 && this.currentNode)
} while (this.closeTags.length === 0 && this.openTags.length === 0)
return true
}
@@ -458,58 +526,32 @@ class TreeSitterHighlightIterator {
// Private methods
descendLeft () {
let child
while ((child = this.currentNode.firstChild) && this.currentIndex === child.startIndex) {
this.currentNode = child
while (this.treeCursor.gotoFirstChild()) {
this.currentChildIndex = 0
this.pushOpenTag()
}
}
currentScopeName () {
return this.layer.grammar.scopeMap.get(
return this.languageMode.grammar.scopeMap.get(
this.containingNodeTypes,
this.containingNodeChildIndices,
this.currentNode.isNamed
this.treeCursor.nodeIsNamed
)
}
pushCloseTag () {
const scopeName = this.currentScopeName()
if (scopeName) this.closeTags.push(this.layer.grammar.idForScope(scopeName))
if (scopeName) this.closeTags.push(this.languageMode.grammar.idForScope(scopeName))
this.containingNodeTypes.pop()
this.containingNodeChildIndices.pop()
}
pushOpenTag () {
this.containingNodeTypes.push(this.currentNode.type)
this.containingNodeTypes.push(this.treeCursor.nodeType)
this.containingNodeChildIndices.push(this.currentChildIndex)
const scopeName = this.currentScopeName()
if (scopeName) this.openTags.push(this.layer.grammar.idForScope(scopeName))
}
}
class TreeSitterTextBufferInput {
constructor (buffer) {
this.buffer = buffer
this.position = {row: 0, column: 0}
this.isBetweenCRLF = false
}
seek (offset, position) {
this.position = position
this.isBetweenCRLF = this.position.column > this.buffer.lineLengthForRow(this.position.row)
}
read () {
const endPosition = this.buffer.clipPosition(new Point(this.position.row + 1000, 0))
let text = this.buffer.getTextInRange([this.position, endPosition])
if (this.isBetweenCRLF) {
text = text.slice(1)
this.isBetweenCRLF = false
}
this.position = endPosition
return text
if (scopeName) this.openTags.push(this.languageMode.grammar.idForScope(scopeName))
}
}