Merge pull request #17956 from atom/mb-tree-sitter-fixes

Fix some Tree-sitter bugs found when enabling by default

spec/tree-sitter-language-mode-spec.js

@@ -175,7 +175,6 @@ describe('TreeSitterLanguageMode', () => {
       ])
 
       buffer.append(')')
-      await nextHighlightingUpdate(languageMode)
       expectTokensToEqual(editor, [
         [
           {text: 'a', scopes: ['function']},
@@ -459,6 +458,50 @@ describe('TreeSitterLanguageMode', () => {
       })
     })
 
+    describe('when changes are small enough to be re-parsed synchronously', () => {
+      it('can incorporate multiple consecutive synchronous updates', () => {
+        const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
+          parser: 'tree-sitter-javascript',
+          scopes: {
+            'property_identifier': 'property',
+            'call_expression > identifier': 'function',
+            'call_expression > member_expression > property_identifier': 'method',
+          }
+        })
+
+        const languageMode = new TreeSitterLanguageMode({buffer, grammar})
+        buffer.setLanguageMode(languageMode)
+        buffer.setText('a');
+        expectTokensToEqual(editor, [[
+          {text: 'a', scopes: []},
+        ]])
+
+        buffer.append('.')
+        expectTokensToEqual(editor, [[
+          {text: 'a.', scopes: []},
+        ]])
+
+        buffer.append('b')
+        expectTokensToEqual(editor, [[
+          {text: 'a.', scopes: []},
+          {text: 'b', scopes: ['property']},
+        ]])
+
+        buffer.append('()')
+        expectTokensToEqual(editor, [[
+          {text: 'a.', scopes: []},
+          {text: 'b', scopes: ['method']},
+          {text: '()', scopes: []},
+        ]])
+
+        buffer.delete([[0, 1], [0, 2]])
+        expectTokensToEqual(editor, [[
+          {text: 'ab', scopes: ['function']},
+          {text: '()', scopes: []},
+        ]])
+      })
+    })
+
     describe('injectionPoints and injectionPatterns', () => {
       let jsGrammar, htmlGrammar
 
@@ -526,7 +569,6 @@ describe('TreeSitterLanguageMode', () => {
         const range = buffer.findSync('html')
         buffer.setTextInRange(range, 'xml')
         await nextHighlightingUpdate(languageMode)
-        await nextHighlightingUpdate(languageMode)
 
         expectTokensToEqual(editor, [
           [
@@ -1371,6 +1413,23 @@ describe('TreeSitterLanguageMode', () => {
         'property.name'
       ])
     })
+
+    it('includes the root scope name even when the given position is in trailing whitespace at EOF', () => {
+      const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
+        scopeName: 'source.js',
+        parser: 'tree-sitter-javascript',
+        scopes: {
+          program: 'source.js',
+          property_identifier: 'property.name'
+        }
+      })
+
+      buffer.setText('a; ')
+      buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
+      expect(editor.scopeDescriptorForBufferPosition([0, 3]).getScopesArray()).toEqual([
+        'source.js'
+      ])
+    })
   })
 
   describe('.bufferRangeForScopeAtPosition(selector?, position)', () => {

src/tree-sitter-language-mode.js

@@ -429,6 +429,9 @@ class TreeSitterLanguageMode {
     for (const scope of iterator.getOpenScopeIds()) {
       scopes.push(this.grammar.scopeNameForScopeId(scope, false))
     }
+    if (scopes.length === 0 || scopes[0] !== this.grammar.scopeName) {
+      scopes.unshift(this.grammar.scopeName)
+    }
     return new ScopeDescriptor({scopes})
   }
 
@@ -515,7 +518,9 @@ class LanguageLayer {
   async update (nodeRangeSet) {
     if (!this.currentParsePromise) {
       do {
-        this.currentParsePromise = this._performUpdate(nodeRangeSet)
+        const params = {async: false}
+        this.currentParsePromise = this._performUpdate(nodeRangeSet, params)
+        if (!params.async) break
         await this.currentParsePromise
       } while (this.tree && this.tree.rootNode.hasChanges())
       this.currentParsePromise = null
@@ -532,7 +537,7 @@ class LanguageLayer {
     }
   }
 
-  async _performUpdate (nodeRangeSet) {
+  async _performUpdate (nodeRangeSet, params) {
     let includedRanges = null
     if (nodeRangeSet) {
       includedRanges = nodeRangeSet.getRanges()
@@ -551,7 +556,10 @@ class LanguageLayer {
       this.tree,
       includedRanges
     )
-    if (tree.then) tree = await tree
+    if (tree.then) {
+      params.async = true
+      tree = await tree
+    }
     tree.buffer = this.languageMode.buffer
 
     const changes = this.patchSinceCurrentParseStarted.getChanges()
@@ -590,7 +598,11 @@ class LanguageLayer {
       }
     }
 
-    await this._populateInjections(affectedRange, nodeRangeSet)
+    const injectionPromise = this._populateInjections(affectedRange, nodeRangeSet)
+    if (injectionPromise) {
+      params.async = true
+      return injectionPromise
+    }
   }
 
   _populateInjections (range, nodeRangeSet) {
@@ -651,11 +663,14 @@ class LanguageLayer {
       }
     }
 
-    const promises = []
-    for (const [marker, nodeRangeSet] of markersToUpdate) {
-      promises.push(marker.languageLayer.update(nodeRangeSet))
+    if (markersToUpdate.size > 0) {
+      this.lastUpdateWasAsync = true
+      const promises = []
+      for (const [marker, nodeRangeSet] of markersToUpdate) {
+        promises.push(marker.languageLayer.update(nodeRangeSet))
+      }
+      return Promise.all(promises)
     }
-    return Promise.all(promises)
   }
 
   _treeEditForBufferChange (start, oldEnd, newEnd, oldText, newText) {