From 4f469feee18f2e6b89766fbbff4d3e0a05f559e2 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Tue, 16 Apr 2019 14:25:18 -0700 Subject: [PATCH 1/5] Revert "Revert "Merge pull request #19165 from atom/mb-upgrade-tree-sitter-to-0-14-again"" This reverts commit e10adac4270f8f3e04cdc53dbd04cc12f86d905c. --- package-lock.json | 36 +++----------------------- package.json | 2 +- spec/spec-helper.coffee | 2 ++ spec/tree-sitter-language-mode-spec.js | 6 ++--- src/tree-sitter-language-mode.js | 10 +++---- 5 files changed, 14 insertions(+), 42 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1adf69570..7d60f9ae1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7185,42 +7185,12 @@ "integrity": "sha512-4hjqbObwlh2dLyW4tcz0Ymw0ggoaVDMveUB9w8kFSQScdRLo0gxO9J7WFcUBo+W3C1TLdFIEwNOWebgZZ0RH9Q==" }, "tree-sitter": { - "version": "0.13.23", - "resolved": "https://registry.npmjs.org/tree-sitter/-/tree-sitter-0.13.23.tgz", - "integrity": "sha512-75AiPbMEstv+YK8h4FkAHnmAJ6nNIUj/NFzRvKCHovmwSEKMi8Wc/E/crB4lJnHBOfV/f/DMQjN+e1Y36kagug==", + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/tree-sitter/-/tree-sitter-0.14.0.tgz", + "integrity": "sha512-aBdFQrQWodjA4MEC7JwPw3M0vhcL3BD4Ivj5jmL6Y8aThGdFxjO0YJzo+//yK8O5RHvfA6rgJa5CYZ2z4qzQrw==", "requires": { "nan": "^2.10.0", "prebuild-install": "^5.0.0" - }, - "dependencies": { - "minimist": { - "version": "1.2.0", - "resolved": "http://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", - "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" - }, - "prebuild-install": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-5.2.1.tgz", - "integrity": "sha512-9DAccsInWHB48TBQi2eJkLPE049JuAI6FjIH0oIrij4bpDVEbX6JvlWRAcAAlUqBHhjgq0jNqA3m3bBXWm9v6w==", - "requires": { - "detect-libc": "^1.0.3", - "expand-template": "^1.0.2", - "github-from-package": "0.0.0", - "minimist": "^1.2.0", - "mkdirp": "^0.5.1", - "napi-build-utils": "^1.0.1", - "node-abi": "^2.2.0", - "noop-logger": "^0.1.1", - "npmlog": "^4.0.1", - "os-homedir": "^1.0.1", - "pump": "^2.0.1", - "rc": "^1.2.7", - "simple-get": "^2.7.0", - "tar-fs": "^1.13.0", - "tunnel-agent": "^0.6.0", - "which-pm-runs": "^1.0.0" - } - } } }, "tree-sitter-bash": { diff --git a/package.json b/package.json index 100612d8a..55e4f6252 100644 --- a/package.json +++ b/package.json @@ -158,7 +158,7 @@ "temp": "^0.9.0", "text-buffer": "13.15.1", "timecop": "https://www.atom.io/api/packages/timecop/versions/0.36.2/tarball", - "tree-sitter": "0.13.23", + "tree-sitter": "0.14.0", "tree-sitter-css": "^0.13.7", "tree-view": "https://www.atom.io/api/packages/tree-view/versions/0.226.0/tarball", "typescript-simple": "1.0.0", diff --git a/spec/spec-helper.coffee b/spec/spec-helper.coffee index f4f1f3568..f56f5c8dc 100644 --- a/spec/spec-helper.coffee +++ b/spec/spec-helper.coffee @@ -12,6 +12,7 @@ FindParentDir = require 'find-parent-dir' TextEditor = require '../src/text-editor' TextEditorElement = require '../src/text-editor-element' TextMateLanguageMode = require '../src/text-mate-language-mode' +TreeSitterLanguageMode = require '../src/tree-sitter-language-mode' {clipboard} = require 'electron' jasmineStyle = document.createElement('style') @@ -101,6 +102,7 @@ beforeEach -> # make tokenization synchronous TextMateLanguageMode.prototype.chunkSize = Infinity + TreeSitterLanguageMode.prototype.syncTimeoutMicros = Infinity spyOn(TextMateLanguageMode.prototype, "tokenizeInBackground").andCallFake -> @tokenizeNextChunk() # Without this spy, TextEditor.onDidTokenize callbacks would not be called diff --git a/spec/tree-sitter-language-mode-spec.js b/spec/tree-sitter-language-mode-spec.js index 7cc210c40..a297d033a 100644 --- a/spec/tree-sitter-language-mode-spec.js +++ b/spec/tree-sitter-language-mode-spec.js @@ -414,7 +414,7 @@ describe('TreeSitterLanguageMode', () => { const languageMode = new TreeSitterLanguageMode({ buffer, grammar, - syncOperationLimit: 0 + syncTimeoutMicros: 0 }) buffer.setLanguageMode(languageMode) await nextHighlightingUpdate(languageMode) @@ -797,7 +797,7 @@ describe('TreeSitterLanguageMode', () => { buffer, grammar: htmlGrammar, grammars: atom.grammars, - syncOperationLimit: 0 + syncTimeoutMicros: 0 }) buffer.setLanguageMode(languageMode) @@ -827,7 +827,7 @@ describe('TreeSitterLanguageMode', () => { ) atom.grammars.loadGrammarSync(jsGrammarPath) atom.grammars.assignLanguageMode(buffer, 'source.js') - buffer.getLanguageMode().syncOperationLimit = 0 + buffer.getLanguageMode().syncTimeoutMicros = 0 const initialSeed = Date.now() for (let i = 0, trialCount = 10; i < trialCount; i++) { diff --git a/src/tree-sitter-language-mode.js b/src/tree-sitter-language-mode.js index 3b40f489f..a8baa45de 100644 --- a/src/tree-sitter-language-mode.js +++ b/src/tree-sitter-language-mode.js @@ -24,7 +24,7 @@ class TreeSitterLanguageMode { } } - constructor ({buffer, grammar, config, grammars, syncOperationLimit}) { + constructor ({buffer, grammar, config, grammars, syncTimeoutMicros}) { TreeSitterLanguageMode._patchSyntaxNode() this.id = nextId++ this.buffer = buffer @@ -35,8 +35,8 @@ class TreeSitterLanguageMode { this.rootLanguageLayer = new LanguageLayer(this, grammar) this.injectionsMarkerLayer = buffer.addMarkerLayer() - if (syncOperationLimit != null) { - this.syncOperationLimit = syncOperationLimit + if (syncTimeoutMicros != null) { + this.syncTimeoutMicros = syncTimeoutMicros } this.rootScopeDescriptor = new ScopeDescriptor({scopes: [this.grammar.scopeName]}) @@ -111,7 +111,7 @@ class TreeSitterLanguageMode { const parser = PARSER_POOL.pop() || new Parser() parser.setLanguage(language) const result = parser.parseTextBuffer(this.buffer.buffer, oldTree, { - syncOperationLimit: this.syncOperationLimit, + syncTimeoutMicros: this.syncTimeoutMicros, includedRanges: ranges }) @@ -1225,6 +1225,6 @@ function hasMatchingFoldSpec (specs, node) { }) TreeSitterLanguageMode.LanguageLayer = LanguageLayer -TreeSitterLanguageMode.prototype.syncOperationLimit = 1000 +TreeSitterLanguageMode.prototype.syncTimeoutMicros = 1000 module.exports = TreeSitterLanguageMode From d66bc8747a227d8fa3d29169f57a069c1374faa4 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 6 May 2019 13:28:01 -0700 Subject: [PATCH 2/5] :arrow_up: tree-sitter (prerelease) --- package-lock.json | 6 +++--- package.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7d60f9ae1..7d16076e4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7185,9 +7185,9 @@ "integrity": "sha512-4hjqbObwlh2dLyW4tcz0Ymw0ggoaVDMveUB9w8kFSQScdRLo0gxO9J7WFcUBo+W3C1TLdFIEwNOWebgZZ0RH9Q==" }, "tree-sitter": { - "version": "0.14.0", - "resolved": "https://registry.npmjs.org/tree-sitter/-/tree-sitter-0.14.0.tgz", - "integrity": "sha512-aBdFQrQWodjA4MEC7JwPw3M0vhcL3BD4Ivj5jmL6Y8aThGdFxjO0YJzo+//yK8O5RHvfA6rgJa5CYZ2z4qzQrw==", + "version": "0.15.0-0", + "resolved": "https://registry.npmjs.org/tree-sitter/-/tree-sitter-0.15.0-0.tgz", + "integrity": "sha512-RHtdWLCUZM/x43imRGRA5+A9VjhBBWXTwd8bEzjCWswmPipU5gwtszyRFsAuLVOQStCmy2DZ4uVmvTOsyCoDng==", "requires": { "nan": "^2.10.0", "prebuild-install": "^5.0.0" diff --git a/package.json b/package.json index 55e4f6252..cea9fa547 100644 --- a/package.json +++ b/package.json @@ -158,7 +158,7 @@ "temp": "^0.9.0", "text-buffer": "13.15.1", "timecop": "https://www.atom.io/api/packages/timecop/versions/0.36.2/tarball", - "tree-sitter": "0.14.0", + "tree-sitter": "0.15.0-0", "tree-sitter-css": "^0.13.7", "tree-view": "https://www.atom.io/api/packages/tree-view/versions/0.226.0/tarball", "typescript-simple": "1.0.0", From a3f022883fe720089378b06368ffa076b9a34253 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 6 May 2019 14:05:56 -0700 Subject: [PATCH 3/5] :arrow_up: tree-sitter (prerelease) --- package-lock.json | 15 +++++++++++---- package.json | 2 +- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7d16076e4..ea07634d5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7185,12 +7185,19 @@ "integrity": "sha512-4hjqbObwlh2dLyW4tcz0Ymw0ggoaVDMveUB9w8kFSQScdRLo0gxO9J7WFcUBo+W3C1TLdFIEwNOWebgZZ0RH9Q==" }, "tree-sitter": { - "version": "0.15.0-0", - "resolved": "https://registry.npmjs.org/tree-sitter/-/tree-sitter-0.15.0-0.tgz", - "integrity": "sha512-RHtdWLCUZM/x43imRGRA5+A9VjhBBWXTwd8bEzjCWswmPipU5gwtszyRFsAuLVOQStCmy2DZ4uVmvTOsyCoDng==", + "version": "0.15.0-1", + "resolved": "https://registry.npmjs.org/tree-sitter/-/tree-sitter-0.15.0-1.tgz", + "integrity": "sha512-2FeiKFUZfJ1MxKqJ5GTAIoQ/SqpqwNeUAdKgZi3MS8m368Mb1Cv6+zyQDcrKyrJWLtKmZs15YJrD9nbMUdcUiA==", "requires": { - "nan": "^2.10.0", + "nan": "^2.13.2", "prebuild-install": "^5.0.0" + }, + "dependencies": { + "nan": { + "version": "2.13.2", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.13.2.tgz", + "integrity": "sha512-TghvYc72wlMGMVMluVo9WRJc0mB8KxxF/gZ4YYFy7V2ZQX9l7rgbPg7vjS9mt6U5HXODVFVI2bOduCzwOMv/lw==" + } } }, "tree-sitter-bash": { diff --git a/package.json b/package.json index cea9fa547..92fad7234 100644 --- a/package.json +++ b/package.json @@ -158,7 +158,7 @@ "temp": "^0.9.0", "text-buffer": "13.15.1", "timecop": "https://www.atom.io/api/packages/timecop/versions/0.36.2/tarball", - "tree-sitter": "0.15.0-0", + "tree-sitter": "0.15.0-1", "tree-sitter-css": "^0.13.7", "tree-view": "https://www.atom.io/api/packages/tree-view/versions/0.226.0/tarball", "typescript-simple": "1.0.0", From 59319d639a04db464e46cdfbe58bffba34b5ec47 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 6 May 2019 14:50:38 -0700 Subject: [PATCH 4/5] Update test to expect new rendering of missing tree-sitter nodes --- spec/tree-sitter-language-mode-spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/tree-sitter-language-mode-spec.js b/spec/tree-sitter-language-mode-spec.js index a297d033a..643a0d0c7 100644 --- a/spec/tree-sitter-language-mode-spec.js +++ b/spec/tree-sitter-language-mode-spec.js @@ -143,7 +143,7 @@ describe('TreeSitterLanguageMode', () => { languageMode.tree.rootNode .descendantForPosition(Point(1, 2), Point(1, 6)) .toString() - ).toBe('(declaration (primitive_type) (identifier) (MISSING))') + ).toBe('(declaration (primitive_type) (identifier) (MISSING ";"))') expectTokensToEqual(editor, [ [ From e33ac81167c73513926e7fd4419fc0e1821301de Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 6 May 2019 16:45:05 -0700 Subject: [PATCH 5/5] :arrow_up: tree-sitter (stable release) --- package-lock.json | 6 +++--- package.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index ea07634d5..65f4cb2c1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7185,9 +7185,9 @@ "integrity": "sha512-4hjqbObwlh2dLyW4tcz0Ymw0ggoaVDMveUB9w8kFSQScdRLo0gxO9J7WFcUBo+W3C1TLdFIEwNOWebgZZ0RH9Q==" }, "tree-sitter": { - "version": "0.15.0-1", - "resolved": "https://registry.npmjs.org/tree-sitter/-/tree-sitter-0.15.0-1.tgz", - "integrity": "sha512-2FeiKFUZfJ1MxKqJ5GTAIoQ/SqpqwNeUAdKgZi3MS8m368Mb1Cv6+zyQDcrKyrJWLtKmZs15YJrD9nbMUdcUiA==", + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/tree-sitter/-/tree-sitter-0.15.0.tgz", + "integrity": "sha512-3CDZ7X4hSJgMXOo8WWhYIwS06hFDmI3x6/3rX+i+LP4ykL3UWVt5TgNKYjAkgBxbpFEHGkaSBe5LRQu5LHOQ3A==", "requires": { "nan": "^2.13.2", "prebuild-install": "^5.0.0" diff --git a/package.json b/package.json index 92fad7234..e13e00f25 100644 --- a/package.json +++ b/package.json @@ -158,7 +158,7 @@ "temp": "^0.9.0", "text-buffer": "13.15.1", "timecop": "https://www.atom.io/api/packages/timecop/versions/0.36.2/tarball", - "tree-sitter": "0.15.0-1", + "tree-sitter": "0.15.0", "tree-sitter-css": "^0.13.7", "tree-view": "https://www.atom.io/api/packages/tree-view/versions/0.226.0/tarball", "typescript-simple": "1.0.0",