Merge pull request #17923 from atom/mb-tree-sitter-sync-updates

Update Tree-sitter syntax highlighting synchronously for parses that complete sync
This commit is contained in:
Max Brunsfeld
2018-08-24 11:44:46 -07:00
committed by GitHub
7 changed files with 69 additions and 85 deletions

View File

@@ -1 +0,0 @@
undefined

View File

@@ -1,6 +1,7 @@
const TextEditorRegistry = require('../src/text-editor-registry')
const TextEditor = require('../src/text-editor')
const TextBuffer = require('text-buffer')
const {Point, Range} = TextBuffer
const {it, fit, ffit, fffit} = require('./async-spec-helpers')
const dedent = require('dedent')
@@ -257,19 +258,19 @@ describe('TextEditorRegistry', function () {
describe('when the "tabType" config setting is "auto"', function () {
it('enables or disables soft tabs based on the editor\'s content', async function () {
await initialPackageActivation
await atom.packages.activatePackage('language-javascript')
atom.grammars.assignLanguageMode(editor, 'source.js')
atom.config.set('editor.tabType', 'auto')
registry.maintainConfig(editor)
await initialPackageActivation
const languageMode = editor.getBuffer().getLanguageMode()
editor.setText(dedent`
{
hello;
}
`)
editor.getBuffer().getLanguageMode().retokenizeLines()
let disposable = registry.maintainConfig(editor)
expect(editor.getSoftTabs()).toBe(true)
editor.setText(dedent`
@@ -277,18 +278,17 @@ describe('TextEditorRegistry', function () {
hello;
}
`)
editor.getBuffer().getLanguageMode().retokenizeLines()
disposable.dispose()
disposable = registry.maintainConfig(editor)
expect(editor.getSoftTabs()).toBe(false)
editor.setText(dedent`
editor.setTextInBufferRange(new Range(Point.ZERO, Point.ZERO), dedent`
/*
* Comment with a leading space.
*/
{
${'\t'}hello;
}
` + editor.getText())
editor.getBuffer().getLanguageMode().retokenizeLines()
` + '\n')
disposable.dispose()
disposable = registry.maintainConfig(editor)
expect(editor.getSoftTabs()).toBe(false)
editor.setText(dedent`
@@ -300,8 +300,8 @@ describe('TextEditorRegistry', function () {
hello;
}
`)
editor.getBuffer().getLanguageMode().retokenizeLines()
disposable.dispose()
disposable = registry.maintainConfig(editor)
expect(editor.getSoftTabs()).toBe(false)
editor.setText(dedent`
@@ -313,7 +313,8 @@ describe('TextEditorRegistry', function () {
hello;
}
`)
editor.getBuffer().getLanguageMode().retokenizeLines()
disposable.dispose()
disposable = registry.maintainConfig(editor)
expect(editor.getSoftTabs()).toBe(true)
})
})

View File

@@ -39,7 +39,6 @@ describe('TreeSitterLanguageMode', () => {
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [[
{text: 'aa.', scopes: ['source']},
@@ -69,7 +68,6 @@ describe('TreeSitterLanguageMode', () => {
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [[
{text: 'a', scopes: ['source', 'variable']},
@@ -96,7 +94,6 @@ describe('TreeSitterLanguageMode', () => {
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [
[
@@ -124,7 +121,6 @@ describe('TreeSitterLanguageMode', () => {
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
expect(
languageMode.tree.rootNode.descendantForPosition(Point(1, 2), Point(1, 6)).toString()
@@ -169,7 +165,6 @@ describe('TreeSitterLanguageMode', () => {
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
// missing closing paren
expectTokensToEqual(editor, [
@@ -208,7 +203,6 @@ describe('TreeSitterLanguageMode', () => {
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [
[
@@ -241,7 +235,6 @@ describe('TreeSitterLanguageMode', () => {
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [
[{text: '// abc', scopes: ['comment']}],
[{text: '', scopes: []}],
@@ -254,7 +247,6 @@ describe('TreeSitterLanguageMode', () => {
])
buffer.insert([2, 0], ' ')
await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [
[{text: '// abc', scopes: ['comment']}],
[{text: '', scopes: []}],
@@ -282,7 +274,6 @@ describe('TreeSitterLanguageMode', () => {
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [
[
@@ -324,7 +315,6 @@ describe('TreeSitterLanguageMode', () => {
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
editor.foldBufferRange([[0, 2], [2, 0]])
@@ -361,7 +351,6 @@ describe('TreeSitterLanguageMode', () => {
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [
[
@@ -394,7 +383,6 @@ describe('TreeSitterLanguageMode', () => {
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [
[
@@ -422,7 +410,7 @@ describe('TreeSitterLanguageMode', () => {
buffer.setText('abc;');
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
const languageMode = new TreeSitterLanguageMode({buffer, grammar, syncOperationLimit: 0})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
await new Promise(process.nextTick)
@@ -509,8 +497,6 @@ describe('TreeSitterLanguageMode', () => {
const languageMode = new TreeSitterLanguageMode({buffer, grammar: jsGrammar, grammars: atom.grammars})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [
[
@@ -570,8 +556,6 @@ describe('TreeSitterLanguageMode', () => {
const languageMode = new TreeSitterLanguageMode({buffer, grammar: htmlGrammar, grammars: atom.grammars})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [
[
@@ -608,7 +592,6 @@ describe('TreeSitterLanguageMode', () => {
const languageMode = new TreeSitterLanguageMode({buffer, grammar: jsGrammar, grammars: atom.grammars})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [
[
{text: 'node.', scopes: []},
@@ -683,14 +666,13 @@ describe('TreeSitterLanguageMode', () => {
atom.grammars.addGrammar(htmlGrammar)
buffer.setText('<body>\n<script>\nb(<%= c.d %>)\n</script>\n</body>')
const languageMode = new TreeSitterLanguageMode({buffer, grammar: ejsGrammar, grammars: atom.grammars})
const languageMode = new TreeSitterLanguageMode({
buffer,
grammar: ejsGrammar,
grammars: atom.grammars,
})
buffer.setLanguageMode(languageMode)
// Parse EJS, then HTML and template JS in parallel, then script tag JS
await nextHighlightingUpdate(languageMode)
await nextHighlightingUpdate(languageMode)
await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [
[
{text: '<', scopes: ['html']},
@@ -752,7 +734,12 @@ describe('TreeSitterLanguageMode', () => {
atom.grammars.addGrammar(htmlGrammar)
buffer.setText('<script>\nhello();\n</script>')
const languageMode = new TreeSitterLanguageMode({buffer, grammar: htmlGrammar, grammars: atom.grammars})
const languageMode = new TreeSitterLanguageMode({
buffer,
grammar: htmlGrammar,
grammars: atom.grammars,
syncOperationLimit: 0
})
buffer.setLanguageMode(languageMode)
await promise
@@ -789,7 +776,6 @@ describe('TreeSitterLanguageMode', () => {
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
expect(editor.isFoldableAtBufferRow(0)).toBe(false)
expect(editor.isFoldableAtBufferRow(1)).toBe(true)
@@ -844,7 +830,6 @@ describe('TreeSitterLanguageMode', () => {
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
// Avoid bringing the `else if...` up onto the same screen line as the preceding `if`.
editor.foldBufferRow(1)
@@ -900,7 +885,6 @@ describe('TreeSitterLanguageMode', () => {
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
expect(editor.isFoldableAtBufferRow(0)).toBe(true)
expect(editor.isFoldableAtBufferRow(1)).toBe(false)
@@ -949,7 +933,6 @@ describe('TreeSitterLanguageMode', () => {
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
expect(editor.isFoldableAtBufferRow(0)).toBe(true)
expect(editor.isFoldableAtBufferRow(1)).toBe(false)
@@ -1024,7 +1007,6 @@ describe('TreeSitterLanguageMode', () => {
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
editor.foldBufferRow(3)
expect(getDisplayText(editor)).toBe(dedent `
@@ -1106,7 +1088,6 @@ describe('TreeSitterLanguageMode', () => {
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
// Void elements have only one child
expect(editor.isFoldableAtBufferRow(1)).toBe(false)
@@ -1159,7 +1140,6 @@ describe('TreeSitterLanguageMode', () => {
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
expect(languageMode.tree.rootNode.toString()).toBe(
"(program (if (identifier) " +
@@ -1229,7 +1209,6 @@ describe('TreeSitterLanguageMode', () => {
`)
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
await nextHighlightingUpdate(buffer.getLanguageMode())
editor.foldBufferRow(0)
expect(getDisplayText(editor)).toBe(dedent `
@@ -1289,9 +1268,6 @@ describe('TreeSitterLanguageMode', () => {
const languageMode = new TreeSitterLanguageMode({buffer, grammar: jsGrammar, grammars: atom.grammars})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
await nextHighlightingUpdate(languageMode)
editor.foldBufferRow(2)
expect(getDisplayText(editor)).toBe(
`a = html \`
@@ -1334,7 +1310,6 @@ describe('TreeSitterLanguageMode', () => {
buffer.setText('foo({bar: baz});')
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
await nextHighlightingUpdate(buffer.getLanguageMode())
expect(editor.scopeDescriptorForBufferPosition([0, 'foo({b'.length]).getScopesArray()).toEqual([
'source.js',
'property.name'
@@ -1385,9 +1360,6 @@ describe('TreeSitterLanguageMode', () => {
const languageMode = new TreeSitterLanguageMode({buffer, grammar: htmlGrammar, grammars: atom.grammars})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
await nextHighlightingUpdate(languageMode)
await nextHighlightingUpdate(languageMode)
const position = buffer.findSync('name').start
expect(languageMode.scopeDescriptorForPosition(position).getScopesArray()).toEqual([
@@ -1412,7 +1384,6 @@ describe('TreeSitterLanguageMode', () => {
buffer.setText('foo({bar: baz});')
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
await nextHighlightingUpdate(buffer.getLanguageMode())
expect(editor.bufferRangeForScopeAtPosition(null, [0, 6])).toEqual(
[[0, 5], [0, 8]]
)
@@ -1453,9 +1424,6 @@ describe('TreeSitterLanguageMode', () => {
const languageMode = new TreeSitterLanguageMode({buffer, grammar: htmlGrammar, grammars: atom.grammars})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
await nextHighlightingUpdate(languageMode)
await nextHighlightingUpdate(languageMode)
const nameProperty = buffer.findSync('name')
const {start} = nameProperty
@@ -1475,7 +1443,6 @@ describe('TreeSitterLanguageMode', () => {
buffer.setText('foo({bar: baz});')
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
await nextHighlightingUpdate(buffer.getLanguageMode())
expect(editor.bufferRangeForScopeAtPosition('.property_identifier', [0, 6])).toEqual(
buffer.findSync('bar')
)
@@ -1519,9 +1486,6 @@ describe('TreeSitterLanguageMode', () => {
const languageMode = new TreeSitterLanguageMode({buffer, grammar: htmlGrammar, grammars: atom.grammars})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
await nextHighlightingUpdate(languageMode)
await nextHighlightingUpdate(languageMode)
const nameProperty = buffer.findSync('name')
const {start} = nameProperty
@@ -1564,9 +1528,6 @@ describe('TreeSitterLanguageMode', () => {
const languageMode = new TreeSitterLanguageMode({buffer, grammar: htmlGrammar, grammars: atom.grammars})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
await nextHighlightingUpdate(languageMode)
await nextHighlightingUpdate(languageMode)
const nameProperty = buffer.findSync('name')
const {start} = nameProperty
@@ -1589,7 +1550,6 @@ describe('TreeSitterLanguageMode', () => {
buffer.setText('foo(bar({x: 2}));')
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
expect(languageMode.getSyntaxNodeAtPosition([0, 6]).range).toEqual(
buffer.findSync('bar')
)
@@ -1616,7 +1576,6 @@ describe('TreeSitterLanguageMode', () => {
`)
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
await nextHighlightingUpdate(buffer.getLanguageMode())
editor.setCursorBufferPosition([1, 3])
editor.selectLargerSyntaxNode()
@@ -1674,9 +1633,6 @@ describe('TreeSitterLanguageMode', () => {
const languageMode = new TreeSitterLanguageMode({buffer, grammar: jsGrammar, grammars: atom.grammars})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
await nextHighlightingUpdate(languageMode)
editor.setCursorBufferPosition({row: 0, column: buffer.getText().indexOf('ef()')})
editor.selectLargerSyntaxNode()
expect(editor.getSelectedText()).toBe('def')

View File

@@ -134,7 +134,7 @@ class GrammarRegistry {
}
this.grammarScoresByBuffer.set(buffer, null)
if (grammar.scopeName !== buffer.getLanguageMode().getLanguageId()) {
if (grammar !== buffer.getLanguageMode().grammar) {
buffer.setLanguageMode(this.languageModeForGrammarAndBuffer(grammar, buffer))
}
@@ -161,7 +161,7 @@ class GrammarRegistry {
)
this.languageOverridesByBufferId.delete(buffer.id)
this.grammarScoresByBuffer.set(buffer, result.score)
if (result.grammar.scopeName !== buffer.getLanguageMode().getLanguageId()) {
if (result.grammar !== buffer.getLanguageMode().grammar) {
buffer.setLanguageMode(this.languageModeForGrammarAndBuffer(result.grammar, buffer))
}
}

View File

@@ -218,7 +218,7 @@ class TextEditorRegistry {
async updateAndMonitorEditorSettings (editor, oldLanguageMode) {
await this.initialPackageActivationPromise
this.updateEditorSettingsForLanguageMode(editor, oldLanguageMode)
await this.subscribeToSettingsForEditorScope(editor)
this.subscribeToSettingsForEditorScope(editor)
}
updateEditorSettingsForLanguageMode (editor, oldLanguageMode) {
@@ -246,7 +246,9 @@ class TextEditorRegistry {
}
}
async subscribeToSettingsForEditorScope (editor) {
subscribeToSettingsForEditorScope (editor) {
if (!this.editorsWithMaintainedConfig) return
const scopeDescriptor = editor.getRootScopeDescriptor()
const scopeChain = scopeDescriptor.getScopeChain()

View File

@@ -4843,7 +4843,7 @@ class TextEditor {
let endRow = bufferRow
const rowCount = this.getLineCount()
while (endRow < rowCount) {
while (endRow + 1 < rowCount) {
if (!NON_WHITESPACE_REGEXP.test(this.lineTextForBufferRow(endRow + 1))) break
if (languageMode.isRowCommented(endRow + 1) !== isCommented) break
endRow++

View File

@@ -23,7 +23,7 @@ class TreeSitterLanguageMode {
}
}
constructor ({buffer, grammar, config, grammars}) {
constructor ({buffer, grammar, config, grammars, syncOperationLimit}) {
TreeSitterLanguageMode._patchSyntaxNode()
this.id = nextId++
this.buffer = buffer
@@ -34,6 +34,10 @@ class TreeSitterLanguageMode {
this.rootLanguageLayer = new LanguageLayer(this, grammar)
this.injectionsMarkerLayer = buffer.addMarkerLayer()
if (syncOperationLimit != null) {
this.syncOperationLimit = syncOperationLimit
}
this.rootScopeDescriptor = new ScopeDescriptor({scopes: [this.grammar.scopeName]})
this.emitter = new Emitter()
this.isFoldableCache = []
@@ -83,15 +87,23 @@ class TreeSitterLanguageMode {
}
}
async parse (language, oldTree, ranges) {
parse (language, oldTree, ranges) {
const parser = PARSER_POOL.pop() || new Parser()
parser.setLanguage(language)
const newTree = await parser.parseTextBuffer(this.buffer.buffer, oldTree, {
syncOperationLimit: 1000,
const result = parser.parseTextBuffer(this.buffer.buffer, oldTree, {
syncOperationLimit: this.syncOperationLimit,
includedRanges: ranges
})
PARSER_POOL.push(parser)
return newTree
if (result.then) {
return result.then(tree => {
PARSER_POOL.push(parser)
return tree
})
} else {
PARSER_POOL.push(parser)
return result
}
}
get tree () {
@@ -107,6 +119,7 @@ class TreeSitterLanguageMode {
*/
buildHighlightIterator () {
if (!this.rootLanguageLayer) return new NullHighlightIterator()
const layerIterators = [
this.rootLanguageLayer.buildHighlightIterator(),
...this.injectionsMarkerLayer.getMarkers().map(m => m.languageLayer.buildHighlightIterator())
@@ -134,7 +147,15 @@ class TreeSitterLanguageMode {
return this.grammar.commentStrings
}
isRowCommented () {
isRowCommented (row) {
const firstNonWhitespaceRange = this.buffer.findInRangeSync(
/\S/,
new Range(new Point(row, 0), new Point(row, Infinity))
)
if (firstNonWhitespaceRange) {
const firstNode = this.getSyntaxNodeContainingRange(firstNonWhitespaceRange)
if (firstNode) return firstNode.type.includes('comment')
}
return false
}
@@ -265,7 +286,9 @@ class TreeSitterLanguageMode {
}
_forEachTreeWithRange (range, callback) {
callback(this.rootLanguageLayer.tree, this.rootLanguageLayer.grammar)
if (this.rootLanguageLayer.tree) {
callback(this.rootLanguageLayer.tree, this.rootLanguageLayer.grammar)
}
const injectionMarkers = this.injectionsMarkerLayer.findMarkers({
intersectsRange: range
@@ -523,11 +546,12 @@ class LanguageLayer {
this.editedRange = null
this.patchSinceCurrentParseStarted = new Patch()
const tree = await this.languageMode.parse(
let tree = this.languageMode.parse(
this.grammar.languageModule,
this.tree,
includedRanges
)
if (tree.then) tree = await tree
tree.buffer = this.languageMode.buffer
const changes = this.patchSinceCurrentParseStarted.getChanges()
@@ -1061,11 +1085,13 @@ function hasMatchingFoldSpec (specs, node) {
'increaseIndentRegexForScopeDescriptor',
'decreaseIndentRegexForScopeDescriptor',
'decreaseNextIndentRegexForScopeDescriptor',
'regexForPattern'
'regexForPattern',
'getNonWordCharacters'
].forEach(methodName => {
TreeSitterLanguageMode.prototype[methodName] = TextMateLanguageMode.prototype[methodName]
})
TreeSitterLanguageMode.LanguageLayer = LanguageLayer
TreeSitterLanguageMode.prototype.syncOperationLimit = 1000
module.exports = TreeSitterLanguageMode