github/atom
1
1
Fork 0
mirror of https://github.com/atom/atom.git synced 2026-01-25 14:59:03 -05:00
Code Issues Packages Projects Releases Wiki Activity

Start work on async parsing

Browse Source
This commit is contained in:
Max Brunsfeld
2018-05-15 17:53:47 -07:00
parent a66120aed2
commit aced30da1f
3 changed files with 100 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
package.json
View File

@@ -72,7 +72,7 @@
"sinon": "1.17.4",
"temp": "^0.8.3",
"text-buffer": "13.14.3",
"tree-sitter": "0.12.1-0",
"tree-sitter": "0.12.1-1",
"typescript-simple": "1.0.0",
"underscore-plus": "^1.6.6",
"winreg": "^1.2.1",

76
spec/tree-sitter-language-mode-spec.js
View File

@@ -20,7 +20,7 @@ describe('TreeSitterLanguageMode', () => {
})
describe('highlighting', () => {
it('applies the most specific scope mapping to each node in the syntax tree', () => {
it('applies the most specific scope mapping to each node in the syntax tree', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
scopes: {
@@ -31,8 +31,11 @@ describe('TreeSitterLanguageMode', () => {
}
})
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText('aa.bbb = cc(d.eee());')
await languageMode.reparsePromise
expectTokensToEqual(editor, [[
{text: 'aa.', scopes: ['source']},
{text: 'bbb', scopes: ['source', 'property']},
@@ -44,7 +47,7 @@ describe('TreeSitterLanguageMode', () => {
]])
})
it('can start or end multiple scopes at the same position', () => {
it('can start or end multiple scopes at the same position', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
scopes: {
@@ -57,8 +60,11 @@ describe('TreeSitterLanguageMode', () => {
}
})
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText('a = bb.ccc();')
await languageMode.reparsePromise
expectTokensToEqual(editor, [[
{text: 'a', scopes: ['source', 'variable']},
{text: ' = ', scopes: ['source']},
@@ -70,7 +76,7 @@ describe('TreeSitterLanguageMode', () => {
]])
})
it('can resume highlighting on a line that starts with whitespace', () => {
it('can resume highlighting on a line that starts with whitespace', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
scopes: {
@@ -80,8 +86,11 @@ describe('TreeSitterLanguageMode', () => {
}
})
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText('a\n .b();')
await languageMode.reparsePromise
expectTokensToEqual(editor, [
[
{text: 'a', scopes: ['variable']},
@@ -95,7 +104,7 @@ describe('TreeSitterLanguageMode', () => {
])
})
it('correctly skips over tokens with zero size', () => {
it('correctly skips over tokens with zero size', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-c',
scopes: {
@@ -107,10 +116,11 @@ describe('TreeSitterLanguageMode', () => {
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText('int main() {\n int a\n int b;\n}');
await languageMode.reparsePromise
editor.screenLineForScreenRow(0)
expect(
languageMode.document.rootNode.descendantForPosition(Point(1, 2), Point(1, 6)).toString()
languageMode.tree.rootNode.descendantForPosition(Point(1, 2), Point(1, 6)).toString()
).toBe('(declaration (primitive_type) (identifier) (MISSING))')
expectTokensToEqual(editor, [
@@ -139,7 +149,7 @@ describe('TreeSitterLanguageMode', () => {
])
})
it('updates lines\' highlighting when they are affected by distant changes', () => {
it('updates lines\' highlighting when they are affected by distant changes', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
scopes: {
@@ -148,10 +158,12 @@ describe('TreeSitterLanguageMode', () => {
}
})
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText('a(\nb,\nc\n')
await languageMode.reparsePromise
// missing closing paren
buffer.setText('a(\nb,\nc\n')
expectTokensToEqual(editor, [
[{text: 'a(', scopes: []}],
[{text: 'b,', scopes: []}],
@@ -160,6 +172,7 @@ describe('TreeSitterLanguageMode', () => {
])
buffer.append(')')
await languageMode.reparsePromise
expectTokensToEqual(editor, [
[
{text: 'a', scopes: ['function']},
@@ -171,7 +184,7 @@ describe('TreeSitterLanguageMode', () => {
])
})
it('handles edits after tokens that end between CR and LF characters (regression)', () => {
it('handles edits after tokens that end between CR and LF characters (regression)', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
scopes: {
@@ -181,13 +194,14 @@ describe('TreeSitterLanguageMode', () => {
}
})
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText([
'// abc',
'',
'a("b").c'
].join('\r\n'))
await languageMode.reparsePromise
expectTokensToEqual(editor, [
[{text: '// abc', scopes: ['comment']}],
@@ -201,6 +215,7 @@ describe('TreeSitterLanguageMode', () => {
])
buffer.insert([2, 0], ' ')
await languageMode.reparsePromise
expectTokensToEqual(editor, [
[{text: '// abc', scopes: ['comment']}],
[{text: '', scopes: []}],
@@ -220,7 +235,7 @@ describe('TreeSitterLanguageMode', () => {
editor.displayLayer.reset({foldCharacter: '…'})
})
it('can fold nodes that start and end with specified tokens', () => {
it('can fold nodes that start and end with specified tokens', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
folds: [
@@ -235,7 +250,8 @@ describe('TreeSitterLanguageMode', () => {
]
})
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText(dedent `
module.exports =
class A {
@@ -246,6 +262,7 @@ describe('TreeSitterLanguageMode', () => {
}
}
`)
await languageMode.reparsePromise
editor.screenLineForScreenRow(0)
@@ -275,7 +292,7 @@ describe('TreeSitterLanguageMode', () => {
`)
})
it('can fold nodes of specified types', () => {
it('can fold nodes of specified types', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
folds: [
@@ -296,7 +313,8 @@ describe('TreeSitterLanguageMode', () => {
]
})
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText(dedent `
const element1 = <Element
className='submit'
@@ -307,6 +325,7 @@ describe('TreeSitterLanguageMode', () => {
<span>world</span>
</Element>
`)
await languageMode.reparsePromise
editor.screenLineForScreenRow(0)
@@ -336,7 +355,7 @@ describe('TreeSitterLanguageMode', () => {
`)
})
it('can fold entire nodes when no start or end parameters are specified', () => {
it('can fold entire nodes when no start or end parameters are specified', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
folds: [
@@ -346,7 +365,8 @@ describe('TreeSitterLanguageMode', () => {
]
})
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText(dedent `
/**
* Important
@@ -355,6 +375,7 @@ describe('TreeSitterLanguageMode', () => {
Also important
*/
`)
await languageMode.reparsePromise
editor.screenLineForScreenRow(0)
@@ -379,7 +400,7 @@ describe('TreeSitterLanguageMode', () => {
`)
})
it('tries each folding strategy for a given node in the order specified', () => {
it('tries each folding strategy for a given node in the order specified', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, cGrammarPath, {
parser: 'tree-sitter-c',
folds: [
@@ -405,8 +426,8 @@ describe('TreeSitterLanguageMode', () => {
]
})
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText(dedent `
#ifndef FOO_H_
#define FOO_H_
@@ -430,6 +451,7 @@ describe('TreeSitterLanguageMode', () => {
#endif
`)
await languageMode.reparsePromise
editor.screenLineForScreenRow(0)
@@ -504,8 +526,6 @@ describe('TreeSitterLanguageMode', () => {
]
})
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
buffer.setText(dedent `
def ab():
print 'a'
@@ -515,6 +535,7 @@ describe('TreeSitterLanguageMode', () => {
print 'c'
print 'd'
`)
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
editor.screenLineForScreenRow(0)
@@ -537,9 +558,8 @@ describe('TreeSitterLanguageMode', () => {
parser: 'tree-sitter-javascript'
})
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
buffer.setText('foo({bar: baz});')
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
editor.screenLineForScreenRow(0)
expect(editor.scopeDescriptorForBufferPosition([0, 6]).getScopesArray()).toEqual([
@@ -562,13 +582,13 @@ describe('TreeSitterLanguageMode', () => {
scopes: {'program': 'source'}
})
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
buffer.setText(dedent `
function a (b, c, d) {
eee.f()
g()
}
`)
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
editor.screenLineForScreenRow(0)

126
src/tree-sitter-language-mode.js
View File

@@ -1,4 +1,4 @@
const {Document} = require('tree-sitter')
const Parser = require('tree-sitter')
const {Point, Range} = require('text-buffer')
const {Emitter, Disposable} = require('event-kit')
const ScopeDescriptor = require('./scope-descriptor')
@@ -14,13 +14,20 @@ class TreeSitterLanguageMode {
this.buffer = buffer
this.grammar = grammar
this.config = config
this.document = new Document()
this.document.setInput(new TreeSitterTextBufferInput(buffer))
this.document.setLanguage(grammar.languageModule)
this.document.parse()
this.parser = new Parser()
this.parser.setLanguage(grammar.languageModule)
this.tree = this.parser.parseTextBufferSync(this.buffer.buffer)
this.rootScopeDescriptor = new ScopeDescriptor({scopes: [this.grammar.id]})
this.emitter = new Emitter()
this.isFoldableCache = []
this.hasQueuedParse = false
this.buffer.onDidChangeText(async () => {
if (!this.reparsePromise) {
this.reparsePromise = this.reparse().then(() => {
this.reparsePromise = null
})
}
})
// TODO: Remove this once TreeSitterLanguageMode implements its own auto-indentation system. This
// is temporarily needed in order to delegate to the TextMateLanguageMode's auto-indent system.
@@ -36,7 +43,7 @@ class TreeSitterLanguageMode {
const oldEndRow = oldRange.end.row
const newEndRow = newRange.end.row
this.isFoldableCache.splice(startRow, oldEndRow - startRow, ...new Array(newEndRow - startRow))
this.document.edit({
this.tree.edit({
startIndex: this.buffer.characterIndexForPosition(oldRange.start),
lengthRemoved: oldText.length,
lengthAdded: newText.length,
@@ -50,8 +57,10 @@ class TreeSitterLanguageMode {
Section - Highlighting
*/
buildHighlightIterator () {
const invalidatedRanges = this.document.parse()
async reparse () {
const tree = await this.parser.parseTextBuffer(this.buffer.buffer, this.tree)
const invalidatedRanges = tree.getChangedRanges(this.tree)
this.tree = tree
for (let i = 0, n = invalidatedRanges.length; i < n; i++) {
const range = invalidatedRanges[i]
const startRow = range.start.row
@@ -61,6 +70,9 @@ class TreeSitterLanguageMode {
}
this.emitter.emit('did-change-highlighting', range)
}
}
buildHighlightIterator () {
return new TreeSitterHighlightIterator(this)
}
@@ -139,7 +151,7 @@ class TreeSitterLanguageMode {
getFoldableRangesAtIndentLevel (goalLevel) {
let result = []
let stack = [{node: this.document.rootNode, level: 0}]
let stack = [{node: this.tree.rootNode, level: 0}]
while (stack.length > 0) {
const {node, level} = stack.pop()
@@ -183,7 +195,7 @@ class TreeSitterLanguageMode {
}
getFoldableRangeContainingPoint (point, tabLength, existenceOnly = false) {
let node = this.document.rootNode.descendantForPosition(this.buffer.clipPosition(point))
let node = this.tree.rootNode.descendantForPosition(this.buffer.clipPosition(point))
while (node) {
if (existenceOnly && node.startPosition.row < point.row) break
if (node.endPosition.row > point.row) {
@@ -273,7 +285,7 @@ class TreeSitterLanguageMode {
getRangeForSyntaxNodeContainingRange (range) {
const startIndex = this.buffer.characterIndexForPosition(range.start)
const endIndex = this.buffer.characterIndexForPosition(range.end)
let node = this.document.rootNode.descendantForIndex(startIndex, endIndex - 1)
let node = this.tree.rootNode.descendantForIndex(startIndex, endIndex - 1)
while (node && node.startIndex === startIndex && node.endIndex === endIndex) {
node = node.parent
}
@@ -305,7 +317,7 @@ class TreeSitterLanguageMode {
scopeDescriptorForPosition (point) {
point = Point.fromObject(point)
const result = []
let node = this.document.rootNode.descendantForPosition(point)
let node = this.tree.rootNode.descendantForPosition(point)
// Don't include anonymous token types like '(' because they prevent scope chains
// from being parsed as CSS selectors by the `slick` parser. Other css selector
@@ -331,17 +343,17 @@ class TreeSitterLanguageMode {
}
class TreeSitterHighlightIterator {
constructor (layer, document) {
constructor (layer) {
this.layer = layer
this.treeCursor = this.layer.tree.walk()
// Conceptually, the iterator represents a single position in the text. It stores this
// position both as a character index and as a `Point`. This position corresponds to a
// leaf node of the syntax tree, which either contains or follows the iterator's
// textual position. The `currentNode` property represents that leaf node, and
// textual position. The `treeCursor` property points at that leaf node, and
// `currentChildIndex` represents the child index of that leaf node within its parent.
this.currentIndex = null
this.currentPosition = null
this.currentNode = null
this.currentChildIndex = null
// In order to determine which selectors match its current node, the iterator maintains
@@ -358,6 +370,8 @@ class TreeSitterHighlightIterator {
}
seek (targetPosition) {
while (this.treeCursor.gotoParent()) {}
const containingTags = []
this.closeTags.length = 0
@@ -367,33 +381,28 @@ class TreeSitterHighlightIterator {
this.currentPosition = targetPosition
this.currentIndex = this.layer.buffer.characterIndexForPosition(targetPosition)
var node = this.layer.document.rootNode
var childIndex = -1
var nodeContainsTarget = true
for (;;) {
this.currentNode = node
this.currentChildIndex = childIndex
if (!nodeContainsTarget) break
this.containingNodeTypes.push(node.type)
this.containingNodeTypes.push(this.treeCursor.nodeType)
this.containingNodeChildIndices.push(childIndex)
const scopeName = this.currentScopeName()
if (scopeName) {
const id = this.layer.grammar.idForScope(scopeName)
if (this.currentIndex === node.startIndex) {
if (this.currentIndex === this.treeCursor.startIndex) {
this.openTags.push(id)
} else {
containingTags.push(id)
}
}
node = node.firstChildForIndex(this.currentIndex)
if (node) {
if (node.startIndex > this.currentIndex) nodeContainsTarget = false
childIndex = node.childIndex
} else {
break
}
const nextChildIndex = this.treeCursor.gotoFirstChildForIndex(this.currentIndex)
if (nextChildIndex == null) break
if (this.treeCursor.startIndex > this.currentIndex) nodeContainsTarget = false
childIndex = nextChildIndex
}
return containingTags
@@ -403,42 +412,35 @@ class TreeSitterHighlightIterator {
this.closeTags.length = 0
this.openTags.length = 0
if (!this.currentNode) {
this.currentPosition = {row: Infinity, column: Infinity}
return false
}
do {
if (this.currentIndex < this.currentNode.startIndex) {
this.currentIndex = this.currentNode.startIndex
this.currentPosition = this.currentNode.startPosition
if (this.currentIndex < this.treeCursor.startIndex) {
this.currentIndex = this.treeCursor.startIndex
this.currentPosition = this.treeCursor.startPosition
this.pushOpenTag()
this.descendLeft()
} else if (this.currentIndex < this.currentNode.endIndex) {
} else if (this.currentIndex < this.treeCursor.endIndex) {
while (true) {
this.currentIndex = this.currentNode.endIndex
this.currentPosition = this.currentNode.endPosition
this.currentIndex = this.treeCursor.endIndex
this.currentPosition = this.treeCursor.endPosition
this.pushCloseTag()
const {nextSibling} = this.currentNode
if (nextSibling && nextSibling.endIndex > this.currentIndex) {
this.currentNode = nextSibling
if (this.treeCursor.gotoNextSibling()) {
this.currentChildIndex++
if (this.currentIndex === nextSibling.startIndex) {
if (this.currentIndex === this.treeCursor.startIndex) {
this.pushOpenTag()
this.descendLeft()
}
break
} else {
this.currentNode = this.currentNode.parent
this.currentChildIndex = last(this.containingNodeChildIndices)
if (!this.currentNode) break
if (!this.treeCursor.gotoParent()) break
}
}
} else {
this.currentNode = this.currentNode.nextSibling
} else if (!this.treeCursor.gotoNextSibling()) {
this.currentPosition = {row: Infinity, column: Infinity}
break
}
} while (this.closeTags.length === 0 && this.openTags.length === 0 && this.currentNode)
} while (this.closeTags.length === 0 && this.openTags.length === 0)
return true
}
@@ -458,9 +460,7 @@ class TreeSitterHighlightIterator {
// Private methods
descendLeft () {
let child
while ((child = this.currentNode.firstChild) && this.currentIndex === child.startIndex) {
this.currentNode = child
while (this.treeCursor.gotoFirstChild()) {
this.currentChildIndex = 0
this.pushOpenTag()
}
@@ -470,7 +470,7 @@ class TreeSitterHighlightIterator {
return this.layer.grammar.scopeMap.get(
this.containingNodeTypes,
this.containingNodeChildIndices,
this.currentNode.isNamed
this.treeCursor.nodeIsNamed
)
}
@@ -482,37 +482,13 @@ class TreeSitterHighlightIterator {
}
pushOpenTag () {
this.containingNodeTypes.push(this.currentNode.type)
this.containingNodeTypes.push(this.treeCursor.nodeType)
this.containingNodeChildIndices.push(this.currentChildIndex)
const scopeName = this.currentScopeName()
if (scopeName) this.openTags.push(this.layer.grammar.idForScope(scopeName))
}
}
class TreeSitterTextBufferInput {
constructor (buffer) {
this.buffer = buffer
this.position = {row: 0, column: 0}
this.isBetweenCRLF = false
}
seek (offset, position) {
this.position = position
this.isBetweenCRLF = this.position.column > this.buffer.lineLengthForRow(this.position.row)
}
read () {
const endPosition = this.buffer.clipPosition(new Point(this.position.row + 1000, 0))
let text = this.buffer.getTextInRange([this.position, endPosition])
if (this.isBetweenCRLF) {
text = text.slice(1)
this.isBetweenCRLF = false
}
this.position = endPosition
return text
}
}
function last (array) {
return array[array.length - 1]
}