From 815cd2b2e9ccc976eef95a8ae5d5e0a84f7d983c Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 2 Nov 2018 17:02:29 -0700 Subject: [PATCH] Add randomized test for updating syntax highlighting, fix bugs --- spec/tree-sitter-language-mode-spec.js | 95 ++++++++++++++++++++++++++ src/tree-sitter-language-mode.js | 42 +++++++++--- 2 files changed, 126 insertions(+), 11 deletions(-) diff --git a/spec/tree-sitter-language-mode-spec.js b/spec/tree-sitter-language-mode-spec.js index db0229e9b..5cc9d488d 100644 --- a/spec/tree-sitter-language-mode-spec.js +++ b/spec/tree-sitter-language-mode-spec.js @@ -1,11 +1,15 @@ const {it, fit, ffit, fffit, beforeEach, afterEach} = require('./async-spec-helpers') +const fs = require('fs') +const path = require('path') const dedent = require('dedent') const TextBuffer = require('text-buffer') const {Point} = TextBuffer const TextEditor = require('../src/text-editor') const TreeSitterGrammar = require('../src/tree-sitter-grammar') const TreeSitterLanguageMode = require('../src/tree-sitter-language-mode') +const Random = require('../script/node_modules/random-seed') +const {getRandomBufferRange, buildRandomLines} = require('./helpers/random') const cGrammarPath = require.resolve('language-c/grammars/tree-sitter-c.cson') const pythonGrammarPath = require.resolve('language-python/grammars/tree-sitter-python.cson') @@ -789,6 +793,97 @@ describe('TreeSitterLanguageMode', () => { }) }) + describe('highlighting after random changes', () => { + let originalTimeout + + beforeEach(() => { + originalTimeout = jasmine.getEnv().defaultTimeoutInterval + jasmine.getEnv().defaultTimeoutInterval = 60 * 1000 + }) + + afterEach(() => { + jasmine.getEnv().defaultTimeoutInterval = originalTimeout + }) + + it('matches the highlighting of a freshly-opened editor', async () => { + jasmine.useRealClock() + + const text = fs.readFileSync(path.join(__dirname, 'fixtures', 'sample.js'), 'utf8') + atom.grammars.loadGrammarSync(jsGrammarPath) + atom.grammars.assignLanguageMode(buffer, 'source.js') + buffer.getLanguageMode().syncOperationLimit = 0 + + const initialSeed = Date.now() + for (let i = 0, trial_count = 10; i < trial_count; i++) { + let seed = initialSeed + i + // seed = 1541201470759 + const random = Random(seed) + + // Parse the initial content and render all of the screen lines. + buffer.setText(text) + buffer.clearUndoStack() + await buffer.getLanguageMode().parseCompletePromise() + editor.displayLayer.getScreenLines() + + // Make several random edits. + for (let j = 0, edit_count = 1 + random(4); j < edit_count; j++) { + const editRoll = random(10) + const range = getRandomBufferRange(random, buffer) + + if (editRoll < 2) { + const linesToInsert = buildRandomLines(random, range.getExtent().row + 1) + // console.log('replace', range.toString(), JSON.stringify(linesToInsert)) + buffer.setTextInRange(range, linesToInsert) + } else if (editRoll < 5) { + // console.log('delete', range.toString()) + buffer.delete(range) + } else { + const linesToInsert = buildRandomLines(random, 3) + // console.log('insert', range.start.toString(), JSON.stringify(linesToInsert)) + buffer.insert(range.start, linesToInsert) + } + + // console.log(buffer.getText()) + + // Sometimes, let the parse complete before re-rendering. + // Sometimes re-render and move on before the parse completes. + if (random(2)) await buffer.getLanguageMode().parseCompletePromise() + editor.displayLayer.getScreenLines() + } + + // Revert the edits, because Tree-sitter's error recovery is somewhat path-dependent, + // and we want a state where the tree parse result is guaranteed. + while (buffer.undo()) {} + + // Create a fresh buffer and editor with the same text. + const buffer2 = new TextBuffer(buffer.getText()) + const editor2 = new TextEditor({buffer: buffer2}) + atom.grammars.assignLanguageMode(buffer2, 'source.js') + + // Verify that the the two buffers have the same syntax highlighting. + await buffer.getLanguageMode().parseCompletePromise() + await buffer2.getLanguageMode().parseCompletePromise() + expect(buffer.getLanguageMode().tree.rootNode.toString()).toEqual( + buffer2.getLanguageMode().tree.rootNode.toString(), `Seed: ${seed}` + ) + + for (let j = 0, n = editor.getScreenLineCount(); j < n; j++) { + const tokens1 = editor.tokensForScreenRow(j) + const tokens2 = editor2.tokensForScreenRow(j) + expect(tokens1).toEqual(tokens2, `Seed: ${seed}, screen line: ${j}`) + if (jasmine.getEnv().currentSpec.results().failedCount > 0) { + console.log(tokens1) + console.log(tokens2) + debugger + break + } + } + + if (jasmine.getEnv().currentSpec.results().failedCount > 0) break + } + }) + }) + describe('folding', () => { it('can fold nodes that start and end with specified tokens', async () => { const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, { diff --git a/src/tree-sitter-language-mode.js b/src/tree-sitter-language-mode.js index ae11274e7..4e8a1470e 100644 --- a/src/tree-sitter-language-mode.js +++ b/src/tree-sitter-language-mode.js @@ -45,7 +45,6 @@ class TreeSitterLanguageMode { this.hasQueuedParse = false this.grammarForLanguageString = this.grammarForLanguageString.bind(this) - this.emitRangeUpdate = this.emitRangeUpdate.bind(this) this.subscription = this.buffer.onDidChangeText(({changes}) => { for (let i = 0, {length} = changes; i < length; i++) { @@ -70,6 +69,25 @@ class TreeSitterLanguageMode { this.regexesByPattern = {} } + async parseCompletePromise () { + let done = false + while (!done) { + if (this.rootLanguageLayer.currentParsePromise) { + await this.rootLanguageLayer.currentParsePromises + } else { + done = true + for (const marker of this.injectionsMarkerLayer.getMarkers()) { + if (marker.languageLayer.currentParsePromise) { + done = false + await marker.languageLayer.currentParsePromise + break + } + } + } + await new Promise(resolve => setTimeout(resolve, 0)) + } + } + destroy () { this.injectionsMarkerLayer.destroy() this.subscription.dispose() @@ -548,8 +566,8 @@ class LanguageLayer { if (this.patchSinceCurrentParseStarted) { this.patchSinceCurrentParseStarted.splice( oldRange.start, - oldRange.end, - newRange.end, + oldRange.getExtent(), + newRange.getExtent(), oldText, newText ) @@ -613,10 +631,14 @@ class LanguageLayer { const changes = this.patchSinceCurrentParseStarted.getChanges() this.patchSinceCurrentParseStarted = null - for (let i = changes.length - 1; i >= 0; i--) { - const {oldStart, oldEnd, newEnd, oldText, newText} = changes[i] + for (const {oldStart, newStart, oldEnd, newEnd, oldText, newText} of changes) { + const newExtent = Point.fromObject(newEnd).traversalFrom(newStart) tree.edit(this._treeEditForBufferChange( - oldStart, oldEnd, newEnd, oldText, newText + newStart, + oldEnd, + Point.fromObject(oldStart).traverse(newExtent), + oldText, + newText )) } @@ -655,9 +677,7 @@ class LanguageLayer { } _populateInjections (range, nodeRangeSet) { - const {injectionsMarkerLayer, grammarForLanguageString} = this.languageMode - - const existingInjectionMarkers = injectionsMarkerLayer + const existingInjectionMarkers = this.languageMode.injectionsMarkerLayer .findMarkers({intersectsRange: range}) .filter(marker => marker.parentLanguageLayer === this) @@ -680,7 +700,7 @@ class LanguageLayer { const languageName = injectionPoint.language(node) if (!languageName) continue - const grammar = grammarForLanguageString(languageName) + const grammar = this.languageMode.grammarForLanguageString(languageName) if (!grammar) continue const contentNodes = injectionPoint.content(node) @@ -695,7 +715,7 @@ class LanguageLayer { m.languageLayer.grammar === grammar ) if (!marker) { - marker = injectionsMarkerLayer.markRange(injectionRange) + marker = this.languageMode.injectionsMarkerLayer.markRange(injectionRange) marker.languageLayer = new LanguageLayer(this.languageMode, grammar, injectionPoint.contentChildTypes) marker.parentLanguageLayer = this }