Add randomized test for updating syntax highlighting, fix bugs

spec/tree-sitter-language-mode-spec.js

@@ -1,11 +1,15 @@
 const {it, fit, ffit, fffit, beforeEach, afterEach} = require('./async-spec-helpers')
 
+const fs = require('fs')
+const path = require('path')
 const dedent = require('dedent')
 const TextBuffer = require('text-buffer')
 const {Point} = TextBuffer
 const TextEditor = require('../src/text-editor')
 const TreeSitterGrammar = require('../src/tree-sitter-grammar')
 const TreeSitterLanguageMode = require('../src/tree-sitter-language-mode')
+const Random = require('../script/node_modules/random-seed')
+const {getRandomBufferRange, buildRandomLines} = require('./helpers/random')
 
 const cGrammarPath = require.resolve('language-c/grammars/tree-sitter-c.cson')
 const pythonGrammarPath = require.resolve('language-python/grammars/tree-sitter-python.cson')
@@ -789,6 +793,97 @@ describe('TreeSitterLanguageMode', () => {
     })
   })
 
+  describe('highlighting after random changes', () => {
+    let originalTimeout
+
+    beforeEach(() => {
+      originalTimeout = jasmine.getEnv().defaultTimeoutInterval
+      jasmine.getEnv().defaultTimeoutInterval = 60 * 1000
+    })
+
+    afterEach(() => {
+      jasmine.getEnv().defaultTimeoutInterval = originalTimeout
+    })
+
+    it('matches the highlighting of a freshly-opened editor', async () => {
+      jasmine.useRealClock()
+
+      const text = fs.readFileSync(path.join(__dirname, 'fixtures', 'sample.js'), 'utf8')
+      atom.grammars.loadGrammarSync(jsGrammarPath)
+      atom.grammars.assignLanguageMode(buffer, 'source.js')
+      buffer.getLanguageMode().syncOperationLimit = 0
+
+      const initialSeed = Date.now()
+      for (let i = 0, trial_count = 10; i < trial_count; i++) {
+        let seed = initialSeed + i
+        // seed = 1541201470759
+        const random = Random(seed)
+
+        // Parse the initial content and render all of the screen lines.
+        buffer.setText(text)
+        buffer.clearUndoStack()
+        await buffer.getLanguageMode().parseCompletePromise()
+        editor.displayLayer.getScreenLines()
+
+        // Make several random edits.
+        for (let j = 0, edit_count = 1 + random(4); j < edit_count; j++) {
+          const editRoll = random(10)
+          const range = getRandomBufferRange(random, buffer)
+
+          if (editRoll < 2) {
+            const linesToInsert = buildRandomLines(random, range.getExtent().row + 1)
+            // console.log('replace', range.toString(), JSON.stringify(linesToInsert))
+            buffer.setTextInRange(range, linesToInsert)
+          } else if (editRoll < 5) {
+            // console.log('delete', range.toString())
+            buffer.delete(range)
+          } else {
+            const linesToInsert = buildRandomLines(random, 3)
+            // console.log('insert', range.start.toString(), JSON.stringify(linesToInsert))
+            buffer.insert(range.start, linesToInsert)
+          }
+
+          // console.log(buffer.getText())
+
+          // Sometimes, let the parse complete before re-rendering.
+          // Sometimes re-render and move on before the parse completes.
+          if (random(2)) await buffer.getLanguageMode().parseCompletePromise()
+          editor.displayLayer.getScreenLines()
+        }
+
+        // Revert the edits, because Tree-sitter's error recovery is somewhat path-dependent,
+        // and we want a state where the tree parse result is guaranteed.
+        while (buffer.undo()) {}
+
+        // Create a fresh buffer and editor with the same text.
+        const buffer2 = new TextBuffer(buffer.getText())
+        const editor2 = new TextEditor({buffer: buffer2})
+        atom.grammars.assignLanguageMode(buffer2, 'source.js')
+
+        // Verify that the the two buffers have the same syntax highlighting.
+        await buffer.getLanguageMode().parseCompletePromise()
+        await buffer2.getLanguageMode().parseCompletePromise()
+        expect(buffer.getLanguageMode().tree.rootNode.toString()).toEqual(
+          buffer2.getLanguageMode().tree.rootNode.toString(), `Seed: ${seed}`
+        )
+
+        for (let j = 0, n = editor.getScreenLineCount(); j < n; j++) {
+          const tokens1 = editor.tokensForScreenRow(j)
+          const tokens2 = editor2.tokensForScreenRow(j)
+          expect(tokens1).toEqual(tokens2, `Seed: ${seed}, screen line: ${j}`)
+          if (jasmine.getEnv().currentSpec.results().failedCount > 0) {
+            console.log(tokens1)
+            console.log(tokens2)
+            debugger
+            break
+          }
+        }
+
+        if (jasmine.getEnv().currentSpec.results().failedCount > 0) break
+      }
+    })
+  })
+
   describe('folding', () => {
     it('can fold nodes that start and end with specified tokens', async () => {
       const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {

src/tree-sitter-language-mode.js

@@ -45,7 +45,6 @@ class TreeSitterLanguageMode {
     this.hasQueuedParse = false
 
     this.grammarForLanguageString = this.grammarForLanguageString.bind(this)
-    this.emitRangeUpdate = this.emitRangeUpdate.bind(this)
 
     this.subscription = this.buffer.onDidChangeText(({changes}) => {
       for (let i = 0, {length} = changes; i < length; i++) {
@@ -70,6 +69,25 @@ class TreeSitterLanguageMode {
     this.regexesByPattern = {}
   }
 
+  async parseCompletePromise () {
+    let done = false
+    while (!done) {
+      if (this.rootLanguageLayer.currentParsePromise) {
+        await this.rootLanguageLayer.currentParsePromises
+      } else {
+        done = true
+        for (const marker of this.injectionsMarkerLayer.getMarkers()) {
+          if (marker.languageLayer.currentParsePromise) {
+            done = false
+            await marker.languageLayer.currentParsePromise
+            break
+          }
+        }
+      }
+      await new Promise(resolve => setTimeout(resolve, 0))
+    }
+  }
+
   destroy () {
     this.injectionsMarkerLayer.destroy()
     this.subscription.dispose()
@@ -548,8 +566,8 @@ class LanguageLayer {
     if (this.patchSinceCurrentParseStarted) {
       this.patchSinceCurrentParseStarted.splice(
         oldRange.start,
-        oldRange.end,
-        newRange.end,
+        oldRange.getExtent(),
+        newRange.getExtent(),
         oldText,
         newText
       )
@@ -613,10 +631,14 @@ class LanguageLayer {
 
     const changes = this.patchSinceCurrentParseStarted.getChanges()
     this.patchSinceCurrentParseStarted = null
-    for (let i = changes.length - 1; i >= 0; i--) {
-      const {oldStart, oldEnd, newEnd, oldText, newText} = changes[i]
+    for (const {oldStart, newStart, oldEnd, newEnd, oldText, newText} of changes) {
+      const newExtent = Point.fromObject(newEnd).traversalFrom(newStart)
       tree.edit(this._treeEditForBufferChange(
-        oldStart, oldEnd, newEnd, oldText, newText
+        newStart,
+        oldEnd,
+        Point.fromObject(oldStart).traverse(newExtent),
+        oldText,
+        newText
       ))
     }
 
@@ -655,9 +677,7 @@ class LanguageLayer {
   }
 
   _populateInjections (range, nodeRangeSet) {
-    const {injectionsMarkerLayer, grammarForLanguageString} = this.languageMode
-
-    const existingInjectionMarkers = injectionsMarkerLayer
+    const existingInjectionMarkers = this.languageMode.injectionsMarkerLayer
       .findMarkers({intersectsRange: range})
       .filter(marker => marker.parentLanguageLayer === this)
 
@@ -680,7 +700,7 @@ class LanguageLayer {
         const languageName = injectionPoint.language(node)
         if (!languageName) continue
 
-        const grammar = grammarForLanguageString(languageName)
+        const grammar = this.languageMode.grammarForLanguageString(languageName)
         if (!grammar) continue
 
         const contentNodes = injectionPoint.content(node)
@@ -695,7 +715,7 @@ class LanguageLayer {
           m.languageLayer.grammar === grammar
         )
         if (!marker) {
-          marker = injectionsMarkerLayer.markRange(injectionRange)
+          marker = this.languageMode.injectionsMarkerLayer.markRange(injectionRange)
           marker.languageLayer = new LanguageLayer(this.languageMode, grammar, injectionPoint.contentChildTypes)
           marker.parentLanguageLayer = this
         }