From 03ac8d715e3ed95ecd578846a5d9ed58f2bf0a6c Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 29 Nov 2017 17:01:20 -0800 Subject: [PATCH 01/39] :arrow_up: language-javascript for new tree-sitter version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index dc3313577..1fc94780f 100644 --- a/package.json +++ b/package.json @@ -147,7 +147,7 @@ "language-html": "0.48.3", "language-hyperlink": "0.16.3", "language-java": "0.27.6", - "language-javascript": "0.127.7", + "language-javascript": "0.128.0-0", "language-json": "0.19.1", "language-less": "0.34.1", "language-make": "0.22.3", From 5c1a49fccf291b229cee1c76faf52e590249784d Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 29 Nov 2017 17:14:11 -0800 Subject: [PATCH 02/39] Add initial TreeSitterLanguageMode implementation Much of this is from the tree-sitter-syntax package. Also, add a dependency on the tree-sitter module. --- package.json | 1 + spec/syntax-scope-map-spec.js | 77 ++++++ src/syntax-scope-map.js | 178 +++++++++++++ src/tree-sitter-grammar.js | 74 ++++++ src/tree-sitter-language-mode.js | 416 +++++++++++++++++++++++++++++++ 5 files changed, 746 insertions(+) create mode 100644 spec/syntax-scope-map-spec.js create mode 100644 src/syntax-scope-map.js create mode 100644 src/tree-sitter-grammar.js create mode 100644 src/tree-sitter-language-mode.js diff --git a/package.json b/package.json index 1fc94780f..91cf950b4 100644 --- a/package.json +++ b/package.json @@ -71,6 +71,7 @@ "sinon": "1.17.4", "temp": "^0.8.3", "text-buffer": "13.9.2", + "tree-sitter": "0.7.4", "typescript-simple": "1.0.0", "underscore-plus": "^1.6.6", "winreg": "^1.2.1", diff --git a/spec/syntax-scope-map-spec.js b/spec/syntax-scope-map-spec.js new file mode 100644 index 000000000..61b1bdc7d --- /dev/null +++ b/spec/syntax-scope-map-spec.js @@ -0,0 +1,77 @@ +const SyntaxScopeMap = require('../src/syntax-scope-map') + +describe('SyntaxScopeMap', () => { + it('can match immediate child selectors', () => { + const map = new SyntaxScopeMap({ + 'a > b > c': 'x', + 'b > c': 'y', + 'c': 'z' + }) + + expect(map.get(['a', 'b', 'c'], [0, 0, 0])).toBe('x') + expect(map.get(['d', 'b', 'c'], [0, 0, 0])).toBe('y') + expect(map.get(['d', 'e', 'c'], [0, 0, 0])).toBe('z') + expect(map.get(['e', 'c'], [0, 0, 0])).toBe('z') + expect(map.get(['c'], [0, 0, 0])).toBe('z') + expect(map.get(['d'], [0, 0, 0])).toBe(undefined) + }) + + it('can match :nth-child pseudo-selectors on leaves', () => { + const map = new SyntaxScopeMap({ + 'a > b': 'w', + 'a > b:nth-child(1)': 'x', + 'b': 'y', + 'b:nth-child(2)': 'z' + }) + + expect(map.get(['a', 'b'], [0, 0])).toBe('w') + expect(map.get(['a', 'b'], [0, 1])).toBe('x') + expect(map.get(['a', 'b'], [0, 2])).toBe('w') + expect(map.get(['b'], [0])).toBe('y') + expect(map.get(['b'], [1])).toBe('y') + expect(map.get(['b'], [2])).toBe('z') + }) + + it('can match :nth-child pseudo-selectors on interior nodes', () => { + const map = new SyntaxScopeMap({ + 'b:nth-child(1) > c': 'w', + 'a > b > c': 'x', + 'a > b:nth-child(2) > c': 'y' + }) + + expect(map.get(['b', 'c'], [0, 0])).toBe(undefined) + expect(map.get(['b', 'c'], [1, 0])).toBe('w') + expect(map.get(['a', 'b', 'c'], [1, 0, 0])).toBe('x') + expect(map.get(['a', 'b', 'c'], [1, 2, 0])).toBe('y') + }) + + it('allows anonymous tokens to be referred to by their string value', () => { + const map = new SyntaxScopeMap({ + '"b"': 'w', + 'a > "b"': 'x', + 'a > "b":nth-child(1)': 'y' + }) + + expect(map.get(['b'], [0], true)).toBe(undefined) + expect(map.get(['b'], [0], false)).toBe('w') + expect(map.get(['a', 'b'], [0, 0], false)).toBe('x') + expect(map.get(['a', 'b'], [0, 1], false)).toBe('y') + }) + + it('supports the wildcard selector', () => { + const map = new SyntaxScopeMap({ + '*': 'w', + 'a > *': 'x', + 'a > *:nth-child(1)': 'y', + 'a > *:nth-child(1) > b': 'z' + }) + + expect(map.get(['b'], [0])).toBe('w') + expect(map.get(['c'], [0])).toBe('w') + expect(map.get(['a', 'b'], [0, 0])).toBe('x') + expect(map.get(['a', 'b'], [0, 1])).toBe('y') + expect(map.get(['a', 'c'], [0, 1])).toBe('y') + expect(map.get(['a', 'c', 'b'], [0, 1, 1])).toBe('z') + expect(map.get(['a', 'c', 'b'], [0, 2, 1])).toBe('w') + }) +}) diff --git a/src/syntax-scope-map.js b/src/syntax-scope-map.js new file mode 100644 index 000000000..e000fb647 --- /dev/null +++ b/src/syntax-scope-map.js @@ -0,0 +1,178 @@ +const parser = require('postcss-selector-parser') + +module.exports = +class SyntaxScopeMap { + constructor (scopeNamesBySelector) { + this.namedScopeTable = {} + this.anonymousScopeTable = {} + for (let selector in scopeNamesBySelector) { + this.addSelector(selector, scopeNamesBySelector[selector]) + } + setTableDefaults(this.namedScopeTable) + setTableDefaults(this.anonymousScopeTable) + } + + addSelector (selector, scopeName) { + parser((parseResult) => { + for (let selectorNode of parseResult.nodes) { + let currentTable = null + let currentIndexValue = null + + for (let i = selectorNode.nodes.length - 1; i >= 0; i--) { + const termNode = selectorNode.nodes[i] + + switch (termNode.type) { + case 'tag': + if (!currentTable) currentTable = this.namedScopeTable + if (!currentTable[termNode.value]) currentTable[termNode.value] = {} + currentTable = currentTable[termNode.value] + if (currentIndexValue != null) { + if (!currentTable.indices) currentTable.indices = {} + if (!currentTable.indices[currentIndexValue]) currentTable.indices[currentIndexValue] = {} + currentTable = currentTable.indices[currentIndexValue] + currentIndexValue = null + } + break + + case 'string': + if (!currentTable) currentTable = this.anonymousScopeTable + const value = termNode.value.slice(1, -1) + if (!currentTable[value]) currentTable[value] = {} + currentTable = currentTable[value] + if (currentIndexValue != null) { + if (!currentTable.indices) currentTable.indices = {} + if (!currentTable.indices[currentIndexValue]) currentTable.indices[currentIndexValue] = {} + currentTable = currentTable.indices[currentIndexValue] + currentIndexValue = null + } + break + + case 'universal': + if (currentTable) { + if (!currentTable['*']) currentTable['*'] = {} + currentTable = currentTable['*'] + } else { + if (!this.namedScopeTable['*']) { + this.namedScopeTable['*'] = this.anonymousScopeTable['*'] = {} + } + currentTable = this.namedScopeTable['*'] + } + if (currentIndexValue != null) { + if (!currentTable.indices) currentTable.indices = {} + if (!currentTable.indices[currentIndexValue]) currentTable.indices[currentIndexValue] = {} + currentTable = currentTable.indices[currentIndexValue] + currentIndexValue = null + } + break + + case 'combinator': + if (currentIndexValue != null) { + rejectSelector(selector) + } + + if (termNode.value === '>') { + if (!currentTable.parents) currentTable.parents = {} + currentTable = currentTable.parents + } else { + rejectSelector(selector) + } + break + + case 'pseudo': + if (termNode.value === ':nth-child') { + currentIndexValue = termNode.nodes[0].nodes[0].value + } else { + rejectSelector(selector) + } + break + + default: + rejectSelector(selector) + } + } + + currentTable.scopeName = scopeName + } + }).process(selector) + } + + get (nodeTypes, childIndices, leafIsNamed = true) { + let result + let i = nodeTypes.length - 1 + let currentTable = leafIsNamed + ? this.namedScopeTable[nodeTypes[i]] + : this.anonymousScopeTable[nodeTypes[i]] + + if (!currentTable) currentTable = this.namedScopeTable['*'] + + while (currentTable) { + if (currentTable.indices && currentTable.indices[childIndices[i]]) { + currentTable = currentTable.indices[childIndices[i]] + } + + if (currentTable.scopeName) { + result = currentTable.scopeName + } + + if (i === 0) break + i-- + currentTable = currentTable.parents && ( + currentTable.parents[nodeTypes[i]] || + currentTable.parents['*'] + ) + } + + return result + } +} + +function setTableDefaults (table) { + const defaultTypeTable = table['*'] + + for (let type in table) { + let typeTable = table[type] + if (typeTable === defaultTypeTable) continue + + if (defaultTypeTable) { + mergeTable(typeTable, defaultTypeTable) + } + + if (typeTable.parents) { + setTableDefaults(typeTable.parents) + } + + for (let key in typeTable.indices) { + const indexTable = typeTable.indices[key] + mergeTable(indexTable, typeTable, false) + if (indexTable.parents) { + setTableDefaults(indexTable.parents) + } + } + } +} + +function mergeTable (table, defaultTable, mergeIndices = true) { + if (mergeIndices && defaultTable.indices) { + if (!table.indices) table.indices = {} + for (let key in defaultTable.indices) { + if (!table.indices[key]) table.indices[key] = {} + mergeTable(table.indices[key], defaultTable.indices[key]) + } + } + + if (defaultTable.parents) { + if (!table.parents) table.parents = {} + for (let key in defaultTable.parents) { + if (!table.parents[key]) table.parents[key] = {} + mergeTable(table.parents[key], defaultTable.parents[key]) + } + } + + if (defaultTable.scopeName && !table.scopeName) { + table.scopeName = defaultTable.scopeName + } +} + +function rejectSelector (selector) { + throw new TypeError(`Unsupported selector '${selector}'`) +} diff --git a/src/tree-sitter-grammar.js b/src/tree-sitter-grammar.js new file mode 100644 index 000000000..141e2da5f --- /dev/null +++ b/src/tree-sitter-grammar.js @@ -0,0 +1,74 @@ +const path = require('path') +const SyntaxScopeMap = require('./syntax-scope-map') +const Module = require('module') +const {OnigRegExp} = require('oniguruma') + +module.exports = +class TreeSitterGrammar { + constructor (registry, filePath, params) { + this.registry = registry + this.id = params.id + this.name = params.name + + this.foldConfig = params.folds || {} + if (!this.foldConfig.delimiters) this.foldConfig.delimiters = [] + if (!this.foldConfig.tokens) this.foldConfig.tokens = [] + + this.commentStrings = { + commentStartString: params.comments && params.comments.start, + commentEndString: params.comments && params.comments.end + } + + const scopeSelectors = {} + for (const key of Object.keys(params.scopes)) { + scopeSelectors[key] = params.scopes[key] + .split('.') + .map(s => `syntax--${s}`) + .join(' ') + } + + this.scopeMap = new SyntaxScopeMap(scopeSelectors) + this.fileTypes = params.fileTypes + + // TODO - When we upgrade to a new enough version of node, use `require.resolve` + // with the new `paths` option instead of this private API. + const languageModulePath = Module._resolveFilename(params.parser, { + id: filePath, + filename: filePath, + paths: Module._nodeModulePaths(path.dirname(filePath)) + }) + + this.languageModule = require(languageModulePath) + this.firstLineRegex = new OnigRegExp(params.firstLineMatch) + this.scopesById = new Map() + this.idsByScope = {} + this.nextScopeId = 256 + 1 + this.registration = null + } + + idForScope (scope) { + let id = this.idsByScope[scope] + if (!id) { + id = this.nextScopeId += 2 + this.idsByScope[scope] = id + this.scopesById.set(id, scope) + } + return id + } + + classNameForScopeId (id) { + return this.scopesById.get(id) + } + + get scopeName () { + return this.id + } + + activate () { + this.registration = this.registry.addGrammar(this) + } + + deactivate () { + if (this.registration) this.registration.dispose() + } +} diff --git a/src/tree-sitter-language-mode.js b/src/tree-sitter-language-mode.js new file mode 100644 index 000000000..7d77a99fd --- /dev/null +++ b/src/tree-sitter-language-mode.js @@ -0,0 +1,416 @@ +const {Document} = require('tree-sitter') +const {Point, Range, Emitter} = require('atom') +const ScopeDescriptor = require('./scope-descriptor') +const TokenizedLine = require('./tokenized-line') + +let nextId = 0 + +module.exports = +class TreeSitterLanguageMode { + constructor ({buffer, grammar, config}) { + this.id = nextId++ + this.buffer = buffer + this.grammar = grammar + this.config = config + this.document = new Document() + this.document.setInput(new TreeSitterTextBufferInput(buffer)) + this.document.setLanguage(grammar.languageModule) + this.document.parse() + this.rootScopeDescriptor = new ScopeDescriptor({scopes: [this.grammar.id]}) + this.emitter = new Emitter() + } + + getLanguageId () { + return this.grammar.id + } + + bufferDidChange ({oldRange, newRange, oldText, newText}) { + this.document.edit({ + startIndex: this.buffer.characterIndexForPosition(oldRange.start), + lengthRemoved: oldText.length, + lengthAdded: newText.length, + startPosition: oldRange.start, + extentRemoved: oldRange.getExtent(), + extentAdded: newRange.getExtent() + }) + } + + /* + * Section - Highlighting + */ + + buildHighlightIterator () { + const invalidatedRanges = this.document.parse() + for (let i = 0, n = invalidatedRanges.length; i < n; i++) { + this.emitter.emit('did-change-highlighting', invalidatedRanges[i]) + } + return new TreeSitterHighlightIterator(this) + } + + onDidChangeHighlighting (callback) { + return this.emitter.on('did-change-hightlighting', callback) + } + + classNameForScopeId (scopeId) { + return this.grammar.classNameForScopeId(scopeId) + } + + /* + * Section - Commenting + */ + + commentStringsForPosition () { + return this.grammar.commentStrings + } + + isRowCommented () { + return false + } + + /* + * Section - Indentation + */ + + suggestedIndentForLineAtBufferRow (row, line, tabLength) { + return this.suggestedIndentForBufferRow(row, tabLength) + } + + suggestedIndentForBufferRow (row, tabLength, options) { + let precedingRow + if (!options || options.skipBlankLines !== false) { + precedingRow = this.buffer.previousNonBlankRow(row) + if (precedingRow == null) return 0 + } else { + precedingRow = row - 1 + if (precedingRow < 0) return 0 + } + + return this.indentLevelForLine(this.buffer.lineForRow(precedingRow), tabLength) + } + + suggestedIndentForEditedBufferRow (row) { + return null + } + + indentLevelForLine (line, tabLength = tabLength) { + let indentLength = 0 + for (let i = 0, {length} = line; i < length; i++) { + const char = line[i] + if (char === '\t') { + indentLength += tabLength - (indentLength % tabLength) + } else if (char === ' ') { + indentLength++ + } else { + break + } + } + return indentLength / tabLength + } + + /* + * Section - Folding + */ + + isFoldableAtRow (row) { + return this.getFoldableRangeContainingPoint(Point(row, Infinity), false) != null + } + + getFoldableRanges () { + return this.getFoldableRangesAtIndentLevel(null) + } + + getFoldableRangesAtIndentLevel (goalLevel) { + let result = [] + let stack = [{node: this.document.rootNode, level: 0}] + while (stack.length > 0) { + const {node, level} = stack.pop() + const startRow = node.startPosition.row + const endRow = node.endPosition.row + + let childLevel = level + const range = this.getFoldableRangeForNode(node) + if (range) { + if (goalLevel == null || level === goalLevel) { + let updatedExistingRange = false + for (let i = 0, {length} = result; i < length; i++) { + if (result[i].start.row === range.start.row && + result[i].end.row === range.end.row) { + result[i] = range + updatedExistingRange = true + } + } + if (!updatedExistingRange) result.push(range) + } + childLevel++ + } + + for (let children = node.namedChildren, i = 0, {length} = children; i < length; i++) { + const child = children[i] + const childStartRow = child.startPosition.row + const childEndRow = child.endPosition.row + if (childEndRow > childStartRow) { + if (childStartRow === startRow && childEndRow === endRow) { + stack.push({node: child, level: level}) + } else if (childLevel <= goalLevel || goalLevel == null) { + stack.push({node: child, level: childLevel}) + } + } + } + } + + return result.sort((a, b) => a.start.row - b.start.row) + } + + getFoldableRangeContainingPoint (point, allowPreviousRows = true) { + let node = this.document.rootNode.descendantForPosition(this.buffer.clipPosition(point)) + while (node) { + if (!allowPreviousRows && node.startPosition.row < point.row) break + if (node.endPosition.row > point.row) { + const range = this.getFoldableRangeForNode(node) + if (range) return range + } + node = node.parent + } + } + + getFoldableRangeForNode (node) { + const {firstChild} = node + if (firstChild) { + const {lastChild} = node + + for (let i = 0, n = this.grammar.foldConfig.delimiters.length; i < n; i++) { + const entry = this.grammar.foldConfig.delimiters[i] + if (firstChild.type === entry[0] && lastChild.type === entry[1]) { + let childPrecedingFold = firstChild + + const options = entry[2] + if (options) { + const {children} = node + let childIndexPrecedingFold = options.afterChildCount || 0 + if (options.afterType) { + for (let i = childIndexPrecedingFold, n = children.length; i < n; i++) { + if (children[i].type === options.afterType) { + childIndexPrecedingFold = i + break + } + } + } + childPrecedingFold = children[childIndexPrecedingFold] + } + + let granchildPrecedingFold = childPrecedingFold.lastChild + if (granchildPrecedingFold) { + return Range(granchildPrecedingFold.endPosition, lastChild.startPosition) + } else { + return Range(childPrecedingFold.endPosition, lastChild.startPosition) + } + } + } + } else { + for (let i = 0, n = this.grammar.foldConfig.tokens.length; i < n; i++) { + const foldableToken = this.grammar.foldConfig.tokens[i] + if (node.type === foldableToken[0]) { + const start = node.startPosition + const end = node.endPosition + start.column += foldableToken[1] + end.column -= foldableToken[2] + return Range(start, end) + } + } + } + } + + /* + * Section - Backward compatibility shims + */ + + tokenizedLineForRow (row) { + return new TokenizedLine({ + openScopes: [], + text: this.buffer.lineForRow(row), + tags: [], + ruleStack: [], + lineEnding: this.buffer.lineEndingForRow(row), + tokenIterator: null, + grammar: this.grammar + }) + } + + scopeDescriptorForPosition (point) { + return this.rootScopeDescriptor + } + + getGrammar () { + return this.grammar + } +} + +class TreeSitterHighlightIterator { + constructor (layer, document) { + this.layer = layer + this.closeTags = null + this.openTags = null + this.containingNodeTypes = null + this.containingNodeChildIndices = null + this.currentNode = null + this.currentChildIndex = null + } + + seek (targetPosition) { + const containingTags = [] + + this.closeTags = [] + this.openTags = [] + this.containingNodeTypes = [] + this.containingNodeChildIndices = [] + this.currentPosition = targetPosition + this.currentIndex = this.layer.buffer.characterIndexForPosition(targetPosition) + + let currentNode = this.layer.document.rootNode + let currentChildIndex = null + while (currentNode) { + this.currentNode = currentNode + this.containingNodeTypes.push(currentNode.type) + this.containingNodeChildIndices.push(currentChildIndex) + + const scopeName = this.currentScopeName() + if (scopeName) { + const id = this.layer.grammar.idForScope(scopeName) + if (this.currentIndex === currentNode.startIndex) { + this.openTags.push(id) + } else { + containingTags.push(id) + } + } + + const {children} = currentNode + currentNode = null + for (let i = 0, childCount = children.length; i < childCount; i++) { + const child = children[i] + if (child.endIndex > this.currentIndex) { + currentNode = child + currentChildIndex = i + break + } + } + } + + return containingTags + } + + moveToSuccessor () { + this.closeTags = [] + this.openTags = [] + + if (!this.currentNode) { + this.currentPosition = {row: Infinity, column: Infinity} + return false + } + + do { + if (this.currentIndex < this.currentNode.endIndex) { + while (true) { + this.pushCloseTag() + const nextSibling = this.currentNode.nextSibling + if (nextSibling) { + if (this.currentNode.endIndex === nextSibling.startIndex) { + this.currentNode = nextSibling + this.currentChildIndex++ + this.currentIndex = nextSibling.startIndex + this.currentPosition = nextSibling.startPosition + this.pushOpenTag() + this.descendLeft() + } else { + this.currentIndex = this.currentNode.endIndex + this.currentPosition = this.currentNode.endPosition + } + break + } else { + this.currentIndex = this.currentNode.endIndex + this.currentPosition = this.currentNode.endPosition + this.currentNode = this.currentNode.parent + this.currentChildIndex = last(this.containingNodeChildIndices) + if (!this.currentNode) break + } + } + } else { + if ((this.currentNode = this.currentNode.nextSibling)) { + this.currentChildIndex++ + this.currentPosition = this.currentNode.startPosition + this.currentIndex = this.currentNode.startIndex + this.pushOpenTag() + this.descendLeft() + } + } + } while (this.closeTags.length === 0 && this.openTags.length === 0 && this.currentNode) + + return true + } + + getPosition () { + return this.currentPosition + } + + getCloseScopeIds () { + return this.closeTags.slice() + } + + getOpenScopeIds () { + return this.openTags.slice() + } + + // Private methods + + descendLeft () { + let child + while ((child = this.currentNode.firstChild)) { + this.currentNode = child + this.currentChildIndex = 0 + this.pushOpenTag() + } + } + + currentScopeName () { + return this.layer.grammar.scopeMap.get( + this.containingNodeTypes, + this.containingNodeChildIndices, + this.currentNode.isNamed + ) + } + + pushCloseTag () { + const scopeName = this.currentScopeName() + if (scopeName) this.closeTags.push(this.layer.grammar.idForScope(scopeName)) + this.containingNodeTypes.pop() + this.containingNodeChildIndices.pop() + } + + pushOpenTag () { + this.containingNodeTypes.push(this.currentNode.type) + this.containingNodeChildIndices.push(this.currentChildIndex) + const scopeName = this.currentScopeName() + if (scopeName) this.openTags.push(this.layer.grammar.idForScope(scopeName)) + } +} + +class TreeSitterTextBufferInput { + constructor (buffer) { + this.buffer = buffer + this.seek(0) + } + + seek (characterIndex) { + this.position = this.buffer.positionForCharacterIndex(characterIndex) + } + + read () { + const endPosition = this.buffer.clipPosition(this.position.traverse({row: 1000, column: 0})) + const text = this.buffer.getTextInRange([this.position, endPosition]) + this.position = endPosition + return text + } +} + +function last (array) { + return array[array.length - 1] +} From 9762685106d161edf4a8df711278da47c170405f Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 29 Nov 2017 17:22:35 -0800 Subject: [PATCH 03/39] Start work on loading tree-sitter grammars in GrammarRegistry --- .../grammars/fake-parser.js | 1 + .../grammars/some-language.cson | 14 ++++ spec/grammar-registry-spec.js | 6 +- spec/package-manager-spec.js | 7 ++ spec/spec-helper.coffee | 3 +- src/grammar-registry.js | 69 +++++++++++++++---- src/tree-sitter-grammar.js | 2 +- 7 files changed, 84 insertions(+), 18 deletions(-) create mode 100644 spec/fixtures/packages/package-with-tree-sitter-grammar/grammars/fake-parser.js create mode 100644 spec/fixtures/packages/package-with-tree-sitter-grammar/grammars/some-language.cson diff --git a/spec/fixtures/packages/package-with-tree-sitter-grammar/grammars/fake-parser.js b/spec/fixtures/packages/package-with-tree-sitter-grammar/grammars/fake-parser.js new file mode 100644 index 000000000..028ee5135 --- /dev/null +++ b/spec/fixtures/packages/package-with-tree-sitter-grammar/grammars/fake-parser.js @@ -0,0 +1 @@ +exports.isFakeTreeSitterParser = true diff --git a/spec/fixtures/packages/package-with-tree-sitter-grammar/grammars/some-language.cson b/spec/fixtures/packages/package-with-tree-sitter-grammar/grammars/some-language.cson new file mode 100644 index 000000000..5eb473456 --- /dev/null +++ b/spec/fixtures/packages/package-with-tree-sitter-grammar/grammars/some-language.cson @@ -0,0 +1,14 @@ +name: 'Some Language' + +id: 'some-language' + +type: 'tree-sitter' + +parser: './fake-parser' + +fileTypes: [ + 'somelang' +] + +scopes: + 'class > identifier': 'entity.name.type.class' diff --git a/spec/grammar-registry-spec.js b/spec/grammar-registry-spec.js index c51ea03b9..3fc5a6056 100644 --- a/spec/grammar-registry-spec.js +++ b/spec/grammar-registry-spec.js @@ -13,8 +13,8 @@ describe('GrammarRegistry', () => { grammarRegistry = new GrammarRegistry({config: atom.config}) }) - describe('.assignLanguageMode(buffer, languageName)', () => { - it('assigns to the buffer a language mode with the given language name', async () => { + describe('.assignLanguageMode(buffer, languageId)', () => { + it('assigns to the buffer a language mode with the given language id', async () => { grammarRegistry.loadGrammarSync(require.resolve('language-javascript/grammars/javascript.cson')) grammarRegistry.loadGrammarSync(require.resolve('language-css/grammars/css.cson')) @@ -34,7 +34,7 @@ describe('GrammarRegistry', () => { expect(buffer.getLanguageMode().getLanguageId()).toBe('source.css') }) - describe('when no languageName is passed', () => { + describe('when no languageId is passed', () => { it('makes the buffer use the null grammar', () => { grammarRegistry.loadGrammarSync(require.resolve('language-css/grammars/css.cson')) diff --git a/spec/package-manager-spec.js b/spec/package-manager-spec.js index 0b26bf839..b1ecf834d 100644 --- a/spec/package-manager-spec.js +++ b/spec/package-manager-spec.js @@ -1030,6 +1030,13 @@ describe('PackageManager', () => { expect(atom.grammars.selectGrammar('a.alot').name).toBe('Alot') expect(atom.grammars.selectGrammar('a.alittle').name).toBe('Alittle') }) + + it('loads any tree-sitter grammars defined in the package', async () => { + await atom.packages.activatePackage('package-with-tree-sitter-grammar') + const grammar = atom.grammars.selectGrammar('test.somelang') + expect(grammar.name).toBe('Some Language') + expect(grammar.languageModule.isFakeTreeSitterParser).toBe(true) + }) }) describe('scoped-property loading', () => { diff --git a/spec/spec-helper.coffee b/spec/spec-helper.coffee index 44319ba52..3bbc78018 100644 --- a/spec/spec-helper.coffee +++ b/spec/spec-helper.coffee @@ -111,7 +111,8 @@ beforeEach -> new CompositeDisposable( @emitter.on("did-tokenize", callback), @onDidChangeGrammar => - if @buffer.getLanguageMode().tokenizeInBackground.originalValue + languageMode = @buffer.getLanguageMode() + if languageMode.tokenizeInBackground?.originalValue callback() ) diff --git a/src/grammar-registry.js b/src/grammar-registry.js index db86958fd..9aa7f1ca6 100644 --- a/src/grammar-registry.js +++ b/src/grammar-registry.js @@ -1,8 +1,11 @@ const _ = require('underscore-plus') const Grim = require('grim') +const CSON = require('season') const FirstMate = require('first-mate') const {Disposable, CompositeDisposable} = require('event-kit') const TextMateLanguageMode = require('./text-mate-language-mode') +const TreeSitterLanguageMode = require('./tree-sitter-language-mode') +const TreeSitterGrammar = require('./tree-sitter-grammar') const Token = require('./token') const fs = require('fs-plus') const {Point, Range} = require('text-buffer') @@ -24,6 +27,7 @@ class GrammarRegistry { clear () { this.textmateRegistry.clear() + this.treeSitterGrammarsById = {} if (this.subscriptions) this.subscriptions.dispose() this.subscriptions = new CompositeDisposable() this.languageOverridesByBufferId = new Map() @@ -112,7 +116,7 @@ class GrammarRegistry { let grammar = null if (languageId != null) { - grammar = this.textmateRegistry.grammarForScopeName(languageId) + grammar = this.grammarForId(languageId) if (!grammar) return false this.languageOverridesByBufferId.set(buffer.id, languageId) } else { @@ -146,7 +150,11 @@ class GrammarRegistry { } languageModeForGrammarAndBuffer (grammar, buffer) { - return new TextMateLanguageMode({grammar, buffer, config: this.config}) + if (grammar instanceof TreeSitterGrammar) { + return new TreeSitterLanguageMode({grammar, buffer, config: this.config}) + } else { + return new TextMateLanguageMode({grammar, buffer, config: this.config}) + } } // Extended: Select a grammar for the given file path and file contents. @@ -165,25 +173,25 @@ class GrammarRegistry { selectGrammarWithScore (filePath, fileContents) { let bestMatch = null let highestScore = -Infinity - for (let grammar of this.textmateRegistry.grammars) { + this.forEachGrammar(grammar => { const score = this.getGrammarScore(grammar, filePath, fileContents) - if ((score > highestScore) || (bestMatch == null)) { + if (score > highestScore || bestMatch == null) { bestMatch = grammar highestScore = score } - } + }) return {grammar: bestMatch, score: highestScore} } // Extended: Returns a {Number} representing how well the grammar matches the // `filePath` and `contents`. getGrammarScore (grammar, filePath, contents) { - if ((contents == null) && fs.isFileSync(filePath)) { + if (contents == null && fs.isFileSync(filePath)) { contents = fs.readFileSync(filePath, 'utf8') } let score = this.getGrammarPathScore(grammar, filePath) - if ((score > 0) && !grammar.bundledPackage) { + if (score > 0 && !grammar.bundledPackage) { score += 0.125 } if (this.grammarMatchesContents(grammar, contents)) { @@ -193,7 +201,7 @@ class GrammarRegistry { } getGrammarPathScore (grammar, filePath) { - if (!filePath) { return -1 } + if (!filePath) return -1 if (process.platform === 'win32') { filePath = filePath.replace(/\\/g, '/') } const pathComponents = filePath.toLowerCase().split(PATH_SPLIT_REGEX) @@ -225,7 +233,7 @@ class GrammarRegistry { } grammarMatchesContents (grammar, contents) { - if ((contents == null) || (grammar.firstLineRegex == null)) { return false } + if (contents == null || grammar.firstLineRegex == null) return false let escaped = false let numberOfNewlinesInRegex = 0 @@ -246,6 +254,20 @@ class GrammarRegistry { return grammar.firstLineRegex.testSync(lines.slice(0, numberOfNewlinesInRegex + 1).join('\n')) } + forEachGrammar (callback) { + this.textmateRegistry.grammars.forEach(callback) + for (let grammarId in this.treeSitterGrammarsById) { + callback(this.treeSitterGrammarsById[grammarId]) + } + } + + grammarForId (languageId) { + return ( + this.textmateRegistry.grammarForScopeName(languageId) || + this.treeSitterGrammarsById[languageId] + ) + } + // Deprecated: Get the grammar override for the given file path. // // * `filePath` A {String} file path. @@ -352,7 +374,13 @@ class GrammarRegistry { } addGrammar (grammar) { - return this.textmateRegistry.addGrammar(grammar) + if (grammar instanceof TreeSitterGrammar) { + this.treeSitterGrammarsById[grammar.id] = grammar + this.grammarAddedOrUpdated(grammar) + return new Disposable(() => delete this.treeSitterGrammarsById[grammar.id]) + } else { + return this.textmateRegistry.addGrammar(grammar) + } } removeGrammar (grammar) { @@ -391,7 +419,15 @@ class GrammarRegistry { // // Returns undefined. readGrammar (grammarPath, callback) { - return this.textmateRegistry.readGrammar(grammarPath, callback) + if (!callback) callback = () => {} + CSON.readFile(grammarPath, (error, params = {}) => { + if (error) return callback(error) + try { + callback(null, this.createGrammar(grammarPath, params)) + } catch (error) { + callback(error) + } + }) } // Extended: Read a grammar synchronously but don't add it to the registry. @@ -400,11 +436,18 @@ class GrammarRegistry { // // Returns a {Grammar}. readGrammarSync (grammarPath) { - return this.textmateRegistry.readGrammarSync(grammarPath) + return this.createGrammar(grammarPath, CSON.readFileSync(grammarPath) || {}) } createGrammar (grammarPath, params) { - return this.textmateRegistry.createGrammar(grammarPath, params) + if (params.type === 'tree-sitter') { + return new TreeSitterGrammar(this, grammarPath, params) + } else { + if (typeof params.scopeName !== 'string' || params.scopeName.length === 0) { + throw new Error(`Grammar missing required scopeName property: ${grammarPath}`) + } + return this.textmateRegistry.createGrammar(grammarPath, params) + } } // Extended: Get all the grammars in this registry. diff --git a/src/tree-sitter-grammar.js b/src/tree-sitter-grammar.js index 141e2da5f..6117f8732 100644 --- a/src/tree-sitter-grammar.js +++ b/src/tree-sitter-grammar.js @@ -39,7 +39,7 @@ class TreeSitterGrammar { }) this.languageModule = require(languageModulePath) - this.firstLineRegex = new OnigRegExp(params.firstLineMatch) + this.firstLineRegex = params.firstLineMatch && new OnigRegExp(params.firstLineMatch) this.scopesById = new Map() this.idsByScope = {} this.nextScopeId = 256 + 1 From 28edfb5b0ae2d995f089ab0454a3300fb0e8fe76 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 29 Nov 2017 17:34:08 -0800 Subject: [PATCH 04/39] Exclude tree-sitter's main JS file from the startup snapshot --- script/lib/generate-startup-snapshot.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/lib/generate-startup-snapshot.js b/script/lib/generate-startup-snapshot.js index 333acdc0a..fd2d049c7 100644 --- a/script/lib/generate-startup-snapshot.js +++ b/script/lib/generate-startup-snapshot.js @@ -57,7 +57,8 @@ module.exports = function (packagedAppPath) { relativePath === path.join('..', 'node_modules', 'spelling-manager', 'node_modules', 'natural', 'lib', 'natural', 'index.js') || relativePath === path.join('..', 'node_modules', 'tar', 'tar.js') || relativePath === path.join('..', 'node_modules', 'temp', 'lib', 'temp.js') || - relativePath === path.join('..', 'node_modules', 'tmp', 'lib', 'tmp.js') + relativePath === path.join('..', 'node_modules', 'tmp', 'lib', 'tmp.js') || + relativePath === path.join('..', 'node_modules', 'tree-sitter', 'index.js') ) } }).then((snapshotScript) => { From 894ce56821a2f6d2427aadab88dba8910249ea84 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 30 Nov 2017 09:32:18 -0800 Subject: [PATCH 05/39] Don't use JS as an example in removeGrammar test --- spec/grammar-registry-spec.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spec/grammar-registry-spec.js b/spec/grammar-registry-spec.js index 3fc5a6056..43fc63d71 100644 --- a/spec/grammar-registry-spec.js +++ b/spec/grammar-registry-spec.js @@ -339,10 +339,10 @@ describe('GrammarRegistry', () => { describe('.removeGrammar(grammar)', () => { it("removes the grammar, so it won't be returned by selectGrammar", async () => { - await atom.packages.activatePackage('language-javascript') - const grammar = atom.grammars.selectGrammar('foo.js') + await atom.packages.activatePackage('language-css') + const grammar = atom.grammars.selectGrammar('foo.css') atom.grammars.removeGrammar(grammar) - expect(atom.grammars.selectGrammar('foo.js').name).not.toBe(grammar.name) + expect(atom.grammars.selectGrammar('foo.css').name).not.toBe(grammar.name) }) }) From 273d708a487408516f011a6db0a7c8c7ccba21f8 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 30 Nov 2017 10:58:26 -0800 Subject: [PATCH 06/39] Add preference for using Tree-sitter parsers --- spec/grammar-registry-spec.js | 73 +++++++++++++++++++++++++++++++- src/config-schema.js | 5 +++ src/grammar-registry.js | 49 ++++++++++++++++++--- src/text-editor.js | 10 +++++ src/tree-sitter-language-mode.js | 4 ++ 5 files changed, 135 insertions(+), 6 deletions(-) diff --git a/spec/grammar-registry-spec.js b/spec/grammar-registry-spec.js index 43fc63d71..4066af24d 100644 --- a/spec/grammar-registry-spec.js +++ b/spec/grammar-registry-spec.js @@ -5,6 +5,8 @@ const fs = require('fs-plus') const temp = require('temp').track() const TextBuffer = require('text-buffer') const GrammarRegistry = require('../src/grammar-registry') +const TreeSitterGrammar = require('../src/tree-sitter-grammar') +const FirstMate = require('first-mate') describe('GrammarRegistry', () => { let grammarRegistry @@ -48,6 +50,30 @@ describe('GrammarRegistry', () => { }) }) + describe('.grammarForId(languageId)', () => { + it('converts the language id to a text-mate language id when `core.useTreeSitterParsers` is false', () => { + atom.config.set('core.useTreeSitterParsers', false) + + grammarRegistry.loadGrammarSync(require.resolve('language-javascript/grammars/javascript.cson')) + grammarRegistry.loadGrammarSync(require.resolve('language-javascript/grammars/tree-sitter-javascript.cson')) + + const grammar = grammarRegistry.grammarForId('javascript') + expect(grammar instanceof FirstMate.Grammar).toBe(true) + expect(grammar.scopeName).toBe('source.js') + }) + + it('converts the language id to a tree-sitter language id when `core.useTreeSitterParsers` is true', () => { + atom.config.set('core.useTreeSitterParsers', true) + + grammarRegistry.loadGrammarSync(require.resolve('language-javascript/grammars/javascript.cson')) + grammarRegistry.loadGrammarSync(require.resolve('language-javascript/grammars/tree-sitter-javascript.cson')) + + const grammar = grammarRegistry.grammarForId('source.js') + expect(grammar instanceof TreeSitterGrammar).toBe(true) + expect(grammar.id).toBe('javascript') + }) + }) + describe('.autoAssignLanguageMode(buffer)', () => { it('assigns to the buffer a language mode based on the best available grammar', () => { grammarRegistry.loadGrammarSync(require.resolve('language-javascript/grammars/javascript.cson')) @@ -78,7 +104,9 @@ describe('GrammarRegistry', () => { expect(buffer.getLanguageMode().getLanguageId()).toBe('source.c') }) - it('updates the buffer\'s grammar when a more appropriate grammar is added for its path', async () => { + it('updates the buffer\'s grammar when a more appropriate text-mate grammar is added for its path', async () => { + atom.config.set('core.useTreeSitterParsers', false) + const buffer = new TextBuffer() expect(buffer.getLanguageMode().getLanguageId()).toBe(null) @@ -87,6 +115,25 @@ describe('GrammarRegistry', () => { grammarRegistry.loadGrammarSync(require.resolve('language-javascript/grammars/javascript.cson')) expect(buffer.getLanguageMode().getLanguageId()).toBe('source.js') + + grammarRegistry.loadGrammarSync(require.resolve('language-javascript/grammars/tree-sitter-javascript.cson')) + expect(buffer.getLanguageMode().getLanguageId()).toBe('source.js') + }) + + it('updates the buffer\'s grammar when a more appropriate tree-sitter grammar is added for its path', async () => { + atom.config.set('core.useTreeSitterParsers', true) + + const buffer = new TextBuffer() + expect(buffer.getLanguageMode().getLanguageId()).toBe(null) + + buffer.setPath('test.js') + grammarRegistry.maintainLanguageMode(buffer) + + grammarRegistry.loadGrammarSync(require.resolve('language-javascript/grammars/tree-sitter-javascript.cson')) + expect(buffer.getLanguageMode().getLanguageId()).toBe('javascript') + + grammarRegistry.loadGrammarSync(require.resolve('language-javascript/grammars/javascript.cson')) + expect(buffer.getLanguageMode().getLanguageId()).toBe('javascript') }) it('can be overridden by calling .assignLanguageMode', () => { @@ -335,6 +382,30 @@ describe('GrammarRegistry', () => { await atom.packages.activatePackage('language-javascript') expect(atom.grammars.selectGrammar('foo.rb', '#!/usr/bin/env node').scopeName).toBe('source.ruby') }) + + describe('tree-sitter vs text-mate', () => { + it('favors a text-mate grammar over a tree-sitter grammar when `core.useTreeSitterParsers` is false', () => { + atom.config.set('core.useTreeSitterParsers', false) + + grammarRegistry.loadGrammarSync(require.resolve('language-javascript/grammars/javascript.cson')) + grammarRegistry.loadGrammarSync(require.resolve('language-javascript/grammars/tree-sitter-javascript.cson')) + + const grammar = grammarRegistry.selectGrammar('test.js') + expect(grammar.scopeName).toBe('source.js') + expect(grammar instanceof FirstMate.Grammar).toBe(true) + }) + + it('favors a tree-sitter grammar over a text-mate grammar when `core.useTreeSitterParsers` is true', () => { + atom.config.set('core.useTreeSitterParsers', true) + + grammarRegistry.loadGrammarSync(require.resolve('language-javascript/grammars/javascript.cson')) + grammarRegistry.loadGrammarSync(require.resolve('language-javascript/grammars/tree-sitter-javascript.cson')) + + const grammar = grammarRegistry.selectGrammar('test.js') + expect(grammar.id).toBe('javascript') + expect(grammar instanceof TreeSitterGrammar).toBe(true) + }) + }) }) describe('.removeGrammar(grammar)', () => { diff --git a/src/config-schema.js b/src/config-schema.js index 2ff68be86..18dc3d774 100644 --- a/src/config-schema.js +++ b/src/config-schema.js @@ -342,6 +342,11 @@ const configSchema = { description: 'Emulated with Atom events' } ] + }, + useTreeSitterParsers: { + type: 'boolean', + default: false, + description: 'Use the new Tree-sitter parsing system for supported languages' } } }, diff --git a/src/grammar-registry.js b/src/grammar-registry.js index 9aa7f1ca6..6dbb248e7 100644 --- a/src/grammar-registry.js +++ b/src/grammar-registry.js @@ -10,9 +10,15 @@ const Token = require('./token') const fs = require('fs-plus') const {Point, Range} = require('text-buffer') +const GRAMMAR_TYPE_BONUS = 1000 const GRAMMAR_SELECTION_RANGE = Range(Point.ZERO, Point(10, 0)).freeze() const PATH_SPLIT_REGEX = new RegExp('[/.]') +const LANGUAGE_ID_MAP = [ + ['source.js', 'javascript'], + ['source.ts', 'typescript'] +] + // Extended: This class holds the grammars used for tokenizing. // // An instance of this class is always available as the `atom.grammars` global. @@ -113,6 +119,7 @@ class GrammarRegistry { // found. assignLanguageMode (buffer, languageId) { if (buffer.getBuffer) buffer = buffer.getBuffer() + languageId = this.normalizeLanguageId(languageId) let grammar = null if (languageId != null) { @@ -197,6 +204,11 @@ class GrammarRegistry { if (this.grammarMatchesContents(grammar, contents)) { score += 0.25 } + + if (score > 0 && this.isGrammarPreferredType(grammar)) { + score += GRAMMAR_TYPE_BONUS + } + return score } @@ -250,6 +262,7 @@ class GrammarRegistry { escaped = false } } + const lines = contents.split('\n') return grammar.firstLineRegex.testSync(lines.slice(0, numberOfNewlinesInRegex + 1).join('\n')) } @@ -262,6 +275,8 @@ class GrammarRegistry { } grammarForId (languageId) { + languageId = this.normalizeLanguageId(languageId) + return ( this.textmateRegistry.grammarForScopeName(languageId) || this.treeSitterGrammarsById[languageId] @@ -306,6 +321,8 @@ class GrammarRegistry { } grammarAddedOrUpdated (grammar) { + if (grammar.scopeName && !grammar.id) grammar.id = grammar.scopeName + this.grammarScoresByBuffer.forEach((score, buffer) => { const languageMode = buffer.getLanguageMode() if (grammar.injectionSelector) { @@ -317,8 +334,8 @@ class GrammarRegistry { const languageOverride = this.languageOverridesByBufferId.get(buffer.id) - if ((grammar.scopeName === buffer.getLanguageMode().getLanguageId() || - grammar.scopeName === languageOverride)) { + if ((grammar.id === buffer.getLanguageMode().getLanguageId() || + grammar.id === languageOverride)) { buffer.setLanguageMode(this.languageModeForGrammarAndBuffer(grammar, buffer)) } else if (!languageOverride) { const score = this.getGrammarScore( @@ -370,7 +387,7 @@ class GrammarRegistry { } grammarForScopeName (scopeName) { - return this.textmateRegistry.grammarForScopeName(scopeName) + return this.grammarForId(scopeName) } addGrammar (grammar) { @@ -398,7 +415,11 @@ class GrammarRegistry { // * `error` An {Error}, may be null. // * `grammar` A {Grammar} or null if an error occured. loadGrammar (grammarPath, callback) { - return this.textmateRegistry.loadGrammar(grammarPath, callback) + this.readGrammar(grammarPath, (error, grammar) => { + if (error) return callback(error) + this.addGrammar(grammar) + callback(grammar) + }) } // Extended: Read a grammar synchronously and add it to this registry. @@ -407,7 +428,9 @@ class GrammarRegistry { // // Returns a {Grammar}. loadGrammarSync (grammarPath) { - return this.textmateRegistry.loadGrammarSync(grammarPath) + const grammar = this.readGrammarSync(grammarPath) + this.addGrammar(grammar) + return grammar } // Extended: Read a grammar asynchronously but don't add it to the registry. @@ -460,4 +483,20 @@ class GrammarRegistry { scopeForId (id) { return this.textmateRegistry.scopeForId(id) } + + isGrammarPreferredType (grammar) { + return this.config.get('core.useTreeSitterParsers') + ? grammar instanceof TreeSitterGrammar + : grammar instanceof FirstMate.Grammar + } + + normalizeLanguageId (languageId) { + if (this.config.get('core.useTreeSitterParsers')) { + const row = LANGUAGE_ID_MAP.find(entry => entry[0] === languageId) + return row ? row[1] : languageId + } else { + const row = LANGUAGE_ID_MAP.find(entry => entry[1] === languageId) + return row ? row[0] : languageId + } + } } diff --git a/src/text-editor.js b/src/text-editor.js index bcd9c19d3..016d076b0 100644 --- a/src/text-editor.js +++ b/src/text-editor.js @@ -3053,6 +3053,16 @@ class TextEditor { return this.expandSelectionsBackward(selection => selection.selectToBeginningOfPreviousParagraph()) } + selectLargerSyntaxNode () { + const languageMode = this.buffer.getLanguageMode() + if (!languageMode.getRangeForSyntaxNodeContainingRange) return + + this.expandSelectionsForward(selection => { + const range = languageMode.getRangeForSyntaxNodeContainingRange(selection.getBufferRange()) + if (range) selection.setBufferRange(range) + }) + } + // Extended: Select the range of the given marker if it is valid. // // * `marker` A {DisplayMarker} diff --git a/src/tree-sitter-language-mode.js b/src/tree-sitter-language-mode.js index 7d77a99fd..0d2e36af6 100644 --- a/src/tree-sitter-language-mode.js +++ b/src/tree-sitter-language-mode.js @@ -240,6 +240,10 @@ class TreeSitterLanguageMode { return this.rootScopeDescriptor } + hasTokenForSelector (scopeSelector) { + return false + } + getGrammar () { return this.grammar } From 203c38ca452cc23751a6714563bbeb80cd320681 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 30 Nov 2017 15:17:14 -0800 Subject: [PATCH 07/39] Add select-{larger,smaller}-syntax-node commands --- keymaps/darwin.cson | 2 ++ src/register-default-commands.coffee | 2 ++ src/text-editor.js | 24 ++++++++++++++++++++++-- src/tree-sitter-language-mode.js | 14 ++++++++++++++ 4 files changed, 40 insertions(+), 2 deletions(-) diff --git a/keymaps/darwin.cson b/keymaps/darwin.cson index 7161a8478..d5cc7b7da 100644 --- a/keymaps/darwin.cson +++ b/keymaps/darwin.cson @@ -161,6 +161,8 @@ 'ctrl-alt-shift-right': 'editor:select-to-next-subword-boundary' 'ctrl-alt-backspace': 'editor:delete-to-beginning-of-subword' 'ctrl-alt-delete': 'editor:delete-to-end-of-subword' + 'ctrl-alt-up': 'editor:select-larger-syntax-node' + 'ctrl-alt-down': 'editor:select-smaller-syntax-node' 'atom-workspace atom-text-editor:not([mini])': # Atom specific diff --git a/src/register-default-commands.coffee b/src/register-default-commands.coffee index 0bacfbb8e..a367e6188 100644 --- a/src/register-default-commands.coffee +++ b/src/register-default-commands.coffee @@ -160,6 +160,8 @@ module.exports = ({commandRegistry, commandInstaller, config, notificationManage 'editor:select-to-previous-subword-boundary': -> @selectToPreviousSubwordBoundary() 'editor:select-to-first-character-of-line': -> @selectToFirstCharacterOfLine() 'editor:select-line': -> @selectLinesContainingCursors() + 'editor:select-larger-syntax-node': -> @selectLargerSyntaxNode() + 'editor:select-smaller-syntax-node': -> @selectSmallerSyntaxNode() }), false ) diff --git a/src/text-editor.js b/src/text-editor.js index 016d076b0..b3d0e592a 100644 --- a/src/text-editor.js +++ b/src/text-editor.js @@ -3053,13 +3053,33 @@ class TextEditor { return this.expandSelectionsBackward(selection => selection.selectToBeginningOfPreviousParagraph()) } + // Extended: For each selection, select the syntax node that contains + // that selection. selectLargerSyntaxNode () { const languageMode = this.buffer.getLanguageMode() if (!languageMode.getRangeForSyntaxNodeContainingRange) return this.expandSelectionsForward(selection => { - const range = languageMode.getRangeForSyntaxNodeContainingRange(selection.getBufferRange()) - if (range) selection.setBufferRange(range) + const currentRange = selection.getBufferRange() + const newRange = languageMode.getRangeForSyntaxNodeContainingRange(currentRange) + if (newRange) { + if (!selection._rangeStack) selection._rangeStack = [] + selection._rangeStack.push(currentRange) + selection.setBufferRange(newRange) + } + }) + } + + // Extended: Undo the effect a preceding call to {::selectLargerSyntaxNode}. + selectSmallerSyntaxNode () { + this.expandSelectionsForward(selection => { + if (selection._rangeStack) { + const lastRange = selection._rangeStack[selection._rangeStack.length - 1] + if (lastRange && selection.getBufferRange().containsRange(lastRange)) { + selection._rangeStack.length-- + selection.setBufferRange(lastRange) + } + } }) } diff --git a/src/tree-sitter-language-mode.js b/src/tree-sitter-language-mode.js index 0d2e36af6..aa2c50a18 100644 --- a/src/tree-sitter-language-mode.js +++ b/src/tree-sitter-language-mode.js @@ -220,6 +220,20 @@ class TreeSitterLanguageMode { } } + /* + * Syntax Tree APIs + */ + + getRangeForSyntaxNodeContainingRange (range) { + const startIndex = this.buffer.characterIndexForPosition(range.start) + const endIndex = this.buffer.characterIndexForPosition(range.end) + let node = this.document.rootNode.descendantForIndex(startIndex, endIndex - 1) + while (node && node.startIndex === startIndex && node.endIndex === endIndex) { + node = node.parent + } + if (node) return new Range(node.startPosition, node.endPosition) + } + /* * Section - Backward compatibility shims */ From 7665c34496b2f1b49ca947be95c863e8d96f8768 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 30 Nov 2017 17:13:30 -0800 Subject: [PATCH 08/39] Start on TreeSitterLanguageMode spec --- spec/tree-sitter-language-mode-spec.js | 61 ++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 spec/tree-sitter-language-mode-spec.js diff --git a/spec/tree-sitter-language-mode-spec.js b/spec/tree-sitter-language-mode-spec.js new file mode 100644 index 000000000..501cdef71 --- /dev/null +++ b/spec/tree-sitter-language-mode-spec.js @@ -0,0 +1,61 @@ +const {it, fit, ffit, fffit, beforeEach, afterEach} = require('./async-spec-helpers') + +const dedent = require('dedent') +const TextBuffer = require('text-buffer') +const TextEditor = require('../src/text-editor') +const TreeSitterGrammar = require('../src/tree-sitter-grammar') +const TreeSitterLanguageMode = require('../src/tree-sitter-language-mode') + +const jsGrammarPath = require.resolve('language-javascript/grammars/tree-sitter-javascript.cson') + +describe('TreeSitterLanguageMode', () => { + let editor, buffer + + beforeEach(async () => { + editor = await atom.workspace.open('') + buffer = editor.getBuffer() + atom.config.set('core.useTreeSitterParsers', true) + }) + + describe('highlighting', () => { + it('applies the most specific scope mapping to each token in the syntax tree', () => { + grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, { + parser: 'tree-sitter-javascript', + scopes: { + 'program': 'source', + 'call_expression > identifier': 'function', + 'property_identifier': 'property', + 'call_expression > member_expression > property_identifier': 'method' + } + }) + + buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar})) + buffer.setText('aa.bbb = cc(d.eee());') + expect(getTokens(editor).slice(0, 1)).toEqual([[ + {text: 'aa.', scopes: ['source']}, + {text: 'bbb', scopes: ['source', 'property']}, + {text: ' = ', scopes: ['source']}, + {text: 'cc', scopes: ['source', 'function']}, + {text: '(d.', scopes: ['source']}, + {text: 'eee', scopes: ['source', 'method']}, + {text: '());', scopes: ['source']} + ]]) + }) + }) +}) + +function getTokens (editor) { + const result = [] + for (let row = 0, lastRow = editor.getLastScreenRow(); row <= lastRow; row++) { + result.push( + editor.tokensForScreenRow(row).map(({text, scopes}) => ({ + text, + scopes: scopes.map(scope => scope + .split(' ') + .map(className => className.slice('syntax--'.length)) + .join(' ')) + })) + ) + } + return result +} From bda50585c4b92ec38fe5944b6a45947f2b75d440 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 1 Dec 2017 14:58:09 -0800 Subject: [PATCH 09/39] Make TreeSitterHighlightIterator stop in between tokens when needed --- src/tree-sitter-language-mode.js | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/src/tree-sitter-language-mode.js b/src/tree-sitter-language-mode.js index aa2c50a18..46da8f4f8 100644 --- a/src/tree-sitter-language-mode.js +++ b/src/tree-sitter-language-mode.js @@ -286,10 +286,12 @@ class TreeSitterHighlightIterator { let currentNode = this.layer.document.rootNode let currentChildIndex = null + let precedesCurrentNode = false while (currentNode) { this.currentNode = currentNode this.containingNodeTypes.push(currentNode.type) this.containingNodeChildIndices.push(currentChildIndex) + if (precedesCurrentNode) break const scopeName = this.currentScopeName() if (scopeName) { @@ -308,6 +310,7 @@ class TreeSitterHighlightIterator { if (child.endIndex > this.currentIndex) { currentNode = child currentChildIndex = i + if (child.startIndex > this.currentIndex) precedesCurrentNode = true break } } @@ -326,33 +329,35 @@ class TreeSitterHighlightIterator { } do { - if (this.currentIndex < this.currentNode.endIndex) { + if (this.currentIndex < this.currentNode.startIndex) { + this.currentIndex = this.currentNode.startIndex + this.currentPosition = this.currentNode.startPosition + this.pushOpenTag() + this.descendLeft() + } else if (this.currentIndex < this.currentNode.endIndex) { while (true) { this.pushCloseTag() - const nextSibling = this.currentNode.nextSibling + this.currentIndex = this.currentNode.endIndex + this.currentPosition = this.currentNode.endPosition + + const {nextSibling} = this.currentNode if (nextSibling) { - if (this.currentNode.endIndex === nextSibling.startIndex) { - this.currentNode = nextSibling - this.currentChildIndex++ - this.currentIndex = nextSibling.startIndex - this.currentPosition = nextSibling.startPosition + this.currentNode = nextSibling + this.currentChildIndex++ + if (this.currentIndex === nextSibling.startIndex) { this.pushOpenTag() this.descendLeft() - } else { - this.currentIndex = this.currentNode.endIndex - this.currentPosition = this.currentNode.endPosition } break } else { - this.currentIndex = this.currentNode.endIndex - this.currentPosition = this.currentNode.endPosition this.currentNode = this.currentNode.parent this.currentChildIndex = last(this.containingNodeChildIndices) if (!this.currentNode) break } } } else { - if ((this.currentNode = this.currentNode.nextSibling)) { + this.currentNode = this.currentNode.nextSibling + if (this.currentNode) { this.currentChildIndex++ this.currentPosition = this.currentNode.startPosition this.currentIndex = this.currentNode.startIndex From d893fb25a8b863d92a35f5a8b156415d8d66e250 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 1 Dec 2017 16:18:25 -0800 Subject: [PATCH 10/39] :art: TreeSitterLanguageMode --- src/tree-sitter-language-mode.js | 74 ++++++++++++++++++++------------ 1 file changed, 46 insertions(+), 28 deletions(-) diff --git a/src/tree-sitter-language-mode.js b/src/tree-sitter-language-mode.js index 46da8f4f8..63fc2a85a 100644 --- a/src/tree-sitter-language-mode.js +++ b/src/tree-sitter-language-mode.js @@ -266,62 +266,80 @@ class TreeSitterLanguageMode { class TreeSitterHighlightIterator { constructor (layer, document) { this.layer = layer - this.closeTags = null - this.openTags = null - this.containingNodeTypes = null - this.containingNodeChildIndices = null + + // Conceptually, the iterator represents a single position in the text. It stores this + // position both as a character index and as a `Point`. This position corresponds to a + // leaf node of the syntax tree, which either contains or follows the iterator's + // textual position. The `currentNode` property represents that leaf node, and + // `currentChildIndex` represents the child index of that leaf node within its parent. + this.currentIndex = null + this.currentPosition = null this.currentNode = null this.currentChildIndex = null + + // In order to determine which selectors match its current node, the iterator maintains + // a list of the current node's ancestors. Because the selectors can use the `:nth-child` + // pseudo-class, each node's child index is also stored. + this.containingNodeTypes = [] + this.containingNodeChildIndices = [] + + // At any given position, the iterator exposes the list of class names that should be + // *ended* at its current position and the list of class names that should be *started* + // at its current position. + this.closeTags = [] + this.openTags = [] } seek (targetPosition) { const containingTags = [] - this.closeTags = [] - this.openTags = [] - this.containingNodeTypes = [] - this.containingNodeChildIndices = [] + this.closeTags.length = 0 + this.openTags.length = 0 + this.containingNodeTypes.length = 0 + this.containingNodeChildIndices.length = 0 this.currentPosition = targetPosition this.currentIndex = this.layer.buffer.characterIndexForPosition(targetPosition) - let currentNode = this.layer.document.rootNode - let currentChildIndex = null - let precedesCurrentNode = false - while (currentNode) { - this.currentNode = currentNode - this.containingNodeTypes.push(currentNode.type) - this.containingNodeChildIndices.push(currentChildIndex) - if (precedesCurrentNode) break + var node = this.layer.document.rootNode + var childIndex = -1 + var done = false + var nodeContainsTarget = true + do { + this.currentNode = node + this.currentChildIndex = childIndex + this.containingNodeTypes.push(node.type) + this.containingNodeChildIndices.push(childIndex) + if (!nodeContainsTarget) break const scopeName = this.currentScopeName() if (scopeName) { const id = this.layer.grammar.idForScope(scopeName) - if (this.currentIndex === currentNode.startIndex) { + if (this.currentIndex === node.startIndex) { this.openTags.push(id) } else { containingTags.push(id) } } - const {children} = currentNode - currentNode = null - for (let i = 0, childCount = children.length; i < childCount; i++) { + done = true + for (var i = 0, {children} = node, childCount = children.length; i < childCount; i++) { const child = children[i] if (child.endIndex > this.currentIndex) { - currentNode = child - currentChildIndex = i - if (child.startIndex > this.currentIndex) precedesCurrentNode = true + node = child + childIndex = i + done = false + if (child.startIndex > this.currentIndex) nodeContainsTarget = false break } } - } + } while (!done) return containingTags } moveToSuccessor () { - this.closeTags = [] - this.openTags = [] + this.closeTags.length = 0 + this.openTags.length = 0 if (!this.currentNode) { this.currentPosition = {row: Infinity, column: Infinity} @@ -336,9 +354,9 @@ class TreeSitterHighlightIterator { this.descendLeft() } else if (this.currentIndex < this.currentNode.endIndex) { while (true) { - this.pushCloseTag() this.currentIndex = this.currentNode.endIndex this.currentPosition = this.currentNode.endPosition + this.pushCloseTag() const {nextSibling} = this.currentNode if (nextSibling) { @@ -386,7 +404,7 @@ class TreeSitterHighlightIterator { descendLeft () { let child - while ((child = this.currentNode.firstChild)) { + while ((child = this.currentNode.firstChild) && this.currentIndex === child.startIndex) { this.currentNode = child this.currentChildIndex = 0 this.pushOpenTag() From 6282cd639a0534eea7728032ce086c0859ff3337 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 4 Dec 2017 10:18:38 -0800 Subject: [PATCH 11/39] Add tree-sitter highlighting test with nested scopes --- spec/tree-sitter-language-mode-spec.js | 49 ++++++++++++++++++++------ 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/spec/tree-sitter-language-mode-spec.js b/spec/tree-sitter-language-mode-spec.js index 501cdef71..c7b1d1f08 100644 --- a/spec/tree-sitter-language-mode-spec.js +++ b/spec/tree-sitter-language-mode-spec.js @@ -14,12 +14,11 @@ describe('TreeSitterLanguageMode', () => { beforeEach(async () => { editor = await atom.workspace.open('') buffer = editor.getBuffer() - atom.config.set('core.useTreeSitterParsers', true) }) describe('highlighting', () => { - it('applies the most specific scope mapping to each token in the syntax tree', () => { - grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, { + it('applies the most specific scope mapping to each node in the syntax tree', () => { + const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, { parser: 'tree-sitter-javascript', scopes: { 'program': 'source', @@ -31,7 +30,7 @@ describe('TreeSitterLanguageMode', () => { buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar})) buffer.setText('aa.bbb = cc(d.eee());') - expect(getTokens(editor).slice(0, 1)).toEqual([[ + expectTokensToEqual(editor, [ {text: 'aa.', scopes: ['source']}, {text: 'bbb', scopes: ['source', 'property']}, {text: ' = ', scopes: ['source']}, @@ -39,16 +38,42 @@ describe('TreeSitterLanguageMode', () => { {text: '(d.', scopes: ['source']}, {text: 'eee', scopes: ['source', 'method']}, {text: '());', scopes: ['source']} - ]]) + ]) + }) + + it('can start or end multiple scopes at the same position', () => { + const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, { + parser: 'tree-sitter-javascript', + scopes: { + 'program': 'source', + 'call_expression': 'call', + 'member_expression': 'member', + 'identifier': 'variable', + '"("': 'open-paren', + '")"': 'close-paren', + } + }) + + buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar})) + buffer.setText('a = bb.ccc();') + expectTokensToEqual(editor, [ + {text: 'a', scopes: ['source', 'variable']}, + {text: ' = ', scopes: ['source']}, + {text: 'bb', scopes: ['source', 'call', 'member', 'variable']}, + {text: '.ccc', scopes: ['source', 'call', 'member']}, + {text: '(', scopes: ['source', 'call', 'open-paren']}, + {text: ')', scopes: ['source', 'call', 'close-paren']}, + {text: ';', scopes: ['source']} + ]) }) }) }) -function getTokens (editor) { - const result = [] +function expectTokensToEqual (editor, expectedTokens) { + const tokens = [] for (let row = 0, lastRow = editor.getLastScreenRow(); row <= lastRow; row++) { - result.push( - editor.tokensForScreenRow(row).map(({text, scopes}) => ({ + tokens.push( + ...editor.tokensForScreenRow(row).map(({text, scopes}) => ({ text, scopes: scopes.map(scope => scope .split(' ') @@ -57,5 +82,9 @@ function getTokens (editor) { })) ) } - return result + + expect(tokens.length).toEqual(expectedTokens.length) + for (let i = 0; i < tokens.length; i++) { + expect(tokens[i]).toEqual(expectedTokens[i], `Token ${i}`) + } } From 4e38b61a5e57ac4f709d7b0fb49ee354c83e6075 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 4 Dec 2017 11:02:24 -0800 Subject: [PATCH 12/39] Optimize TreeSitterLanguageMode.isFoldableAtRow --- spec/tree-sitter-language-mode-spec.js | 61 ++++++++++++++++++++++++++ src/tree-sitter-language-mode.js | 12 ++--- 2 files changed, 68 insertions(+), 5 deletions(-) diff --git a/spec/tree-sitter-language-mode-spec.js b/spec/tree-sitter-language-mode-spec.js index c7b1d1f08..79ca654c7 100644 --- a/spec/tree-sitter-language-mode-spec.js +++ b/spec/tree-sitter-language-mode-spec.js @@ -67,8 +67,69 @@ describe('TreeSitterLanguageMode', () => { ]) }) }) + + describe('folding', () => { + beforeEach(() => { + editor.displayLayer.reset({foldCharacter: '…'}) + }) + + it('folds nodes that start and end with specified tokens and span multiple lines', () => { + const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, { + parser: 'tree-sitter-javascript', + scopes: {'program': 'source'}, + folds: { + delimiters: [ + ['{', '}'], + ['(', ')'] + ] + } + }) + + buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar})) + buffer.setText(dedent ` + module.exports = + class A { + getB (c, + d, + e) { + return this.b + } + } + `) + + editor.screenLineForScreenRow(0) + + expect(editor.isFoldableAtBufferRow(0)).toBe(false) + expect(editor.isFoldableAtBufferRow(1)).toBe(true) + expect(editor.isFoldableAtBufferRow(2)).toBe(true) + expect(editor.isFoldableAtBufferRow(3)).toBe(false) + expect(editor.isFoldableAtBufferRow(4)).toBe(true) + + editor.foldBufferRow(2) + expect(getDisplayText(editor)).toBe(dedent ` + module.exports = + class A { + getB (…) { + return this.b + } + } + `) + + editor.foldBufferRow(4) + expect(getDisplayText(editor)).toBe(dedent ` + module.exports = + class A { + getB (…) {…} + } + `) + }) + }) }) +function getDisplayText (editor) { + return editor.displayLayer.getText() +} + function expectTokensToEqual (editor, expectedTokens) { const tokens = [] for (let row = 0, lastRow = editor.getLastScreenRow(); row <= lastRow; row++) { diff --git a/src/tree-sitter-language-mode.js b/src/tree-sitter-language-mode.js index 63fc2a85a..4c3df538a 100644 --- a/src/tree-sitter-language-mode.js +++ b/src/tree-sitter-language-mode.js @@ -112,7 +112,7 @@ class TreeSitterLanguageMode { */ isFoldableAtRow (row) { - return this.getFoldableRangeContainingPoint(Point(row, Infinity), false) != null + return this.getFoldableRangeContainingPoint(Point(row, Infinity), 0, true) != null } getFoldableRanges () { @@ -161,19 +161,19 @@ class TreeSitterLanguageMode { return result.sort((a, b) => a.start.row - b.start.row) } - getFoldableRangeContainingPoint (point, allowPreviousRows = true) { + getFoldableRangeContainingPoint (point, tabLength, existenceOnly = false) { let node = this.document.rootNode.descendantForPosition(this.buffer.clipPosition(point)) while (node) { - if (!allowPreviousRows && node.startPosition.row < point.row) break + if (existenceOnly && node.startPosition.row < point.row) break if (node.endPosition.row > point.row) { - const range = this.getFoldableRangeForNode(node) + const range = this.getFoldableRangeForNode(node, existenceOnly) if (range) return range } node = node.parent } } - getFoldableRangeForNode (node) { + getFoldableRangeForNode (node, existenceOnly) { const {firstChild} = node if (firstChild) { const {lastChild} = node @@ -181,6 +181,7 @@ class TreeSitterLanguageMode { for (let i = 0, n = this.grammar.foldConfig.delimiters.length; i < n; i++) { const entry = this.grammar.foldConfig.delimiters[i] if (firstChild.type === entry[0] && lastChild.type === entry[1]) { + if (existenceOnly) return true let childPrecedingFold = firstChild const options = entry[2] @@ -210,6 +211,7 @@ class TreeSitterLanguageMode { for (let i = 0, n = this.grammar.foldConfig.tokens.length; i < n; i++) { const foldableToken = this.grammar.foldConfig.tokens[i] if (node.type === foldableToken[0]) { + if (existenceOnly) return true const start = node.startPosition const end = node.endPosition start.column += foldableToken[1] From 98e11673aa37728b00503540dec3cea0c4fe7306 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 4 Dec 2017 11:40:44 -0800 Subject: [PATCH 13/39] Tweak TreeSitterLanguageMode folding configuration --- spec/tree-sitter-language-mode-spec.js | 47 +++++++++++++++++++++++++- src/tree-sitter-grammar.js | 7 ++-- src/tree-sitter-language-mode.js | 21 +++++------- 3 files changed, 59 insertions(+), 16 deletions(-) diff --git a/spec/tree-sitter-language-mode-spec.js b/spec/tree-sitter-language-mode-spec.js index 79ca654c7..93937f4b4 100644 --- a/spec/tree-sitter-language-mode-spec.js +++ b/spec/tree-sitter-language-mode-spec.js @@ -73,7 +73,7 @@ describe('TreeSitterLanguageMode', () => { editor.displayLayer.reset({foldCharacter: '…'}) }) - it('folds nodes that start and end with specified tokens and span multiple lines', () => { + it('can fold nodes that start and end with specified tokens and span multiple lines', () => { const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, { parser: 'tree-sitter-javascript', scopes: {'program': 'source'}, @@ -123,6 +123,51 @@ describe('TreeSitterLanguageMode', () => { } `) }) + + it('can fold specified types of multi-line nodes', () => { + const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, { + parser: 'tree-sitter-javascript', + scopes: {'program': 'source'}, + folds: { + nodes: [ + 'template_string', + 'comment' + ] + } + }) + + buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar})) + buffer.setText(dedent ` + /** + * Important + */ + const x = \`one + two + three\` + `) + + editor.screenLineForScreenRow(0) + + expect(editor.isFoldableAtBufferRow(0)).toBe(true) + expect(editor.isFoldableAtBufferRow(1)).toBe(false) + expect(editor.isFoldableAtBufferRow(2)).toBe(false) + expect(editor.isFoldableAtBufferRow(3)).toBe(true) + expect(editor.isFoldableAtBufferRow(4)).toBe(false) + + editor.foldBufferRow(0) + expect(getDisplayText(editor)).toBe(dedent ` + /**… */ + const x = \`one + two + three\` + `) + + editor.foldBufferRow(3) + expect(getDisplayText(editor)).toBe(dedent ` + /**… */ + const x = \`one… three\` + `) + }) }) }) diff --git a/src/tree-sitter-grammar.js b/src/tree-sitter-grammar.js index 6117f8732..d7d36a0a7 100644 --- a/src/tree-sitter-grammar.js +++ b/src/tree-sitter-grammar.js @@ -10,9 +10,10 @@ class TreeSitterGrammar { this.id = params.id this.name = params.name - this.foldConfig = params.folds || {} - if (!this.foldConfig.delimiters) this.foldConfig.delimiters = [] - if (!this.foldConfig.tokens) this.foldConfig.tokens = [] + this.foldConfig = { + delimiters: params.folds && params.folds.delimiters || [], + nodes: new Set(params.folds && params.folds.nodes || []) + } this.commentStrings = { commentStartString: params.comments && params.comments.start, diff --git a/src/tree-sitter-language-mode.js b/src/tree-sitter-language-mode.js index 4c3df538a..8d4049a51 100644 --- a/src/tree-sitter-language-mode.js +++ b/src/tree-sitter-language-mode.js @@ -207,18 +207,15 @@ class TreeSitterLanguageMode { } } } - } else { - for (let i = 0, n = this.grammar.foldConfig.tokens.length; i < n; i++) { - const foldableToken = this.grammar.foldConfig.tokens[i] - if (node.type === foldableToken[0]) { - if (existenceOnly) return true - const start = node.startPosition - const end = node.endPosition - start.column += foldableToken[1] - end.column -= foldableToken[2] - return Range(start, end) - } - } + } + + if (this.grammar.foldConfig.nodes.has(node.type)) { + if (existenceOnly) return true + const start = node.startPosition + const end = node.endPosition + start.column = Infinity + end.column = 0 + return Range(start, end) } } From 8a1c7619f3063d2935824f6d78721009adbccd24 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Mon, 4 Dec 2017 12:07:05 -0800 Subject: [PATCH 14/39] Add test for .select{Larger,Smaller}SyntaxNode --- spec/tree-sitter-language-mode-spec.js | 42 ++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/spec/tree-sitter-language-mode-spec.js b/spec/tree-sitter-language-mode-spec.js index 93937f4b4..b05147631 100644 --- a/spec/tree-sitter-language-mode-spec.js +++ b/spec/tree-sitter-language-mode-spec.js @@ -169,6 +169,48 @@ describe('TreeSitterLanguageMode', () => { `) }) }) + + describe('TextEditor.selectLargerSyntaxNode and .selectSmallerSyntaxNode', () => { + it('expands and contract the selection based on the syntax tree', () => { + const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, { + parser: 'tree-sitter-javascript', + scopes: {'program': 'source'} + }) + + buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar})) + buffer.setText(dedent ` + function a (b, c, d) { + eee.f() + g() + } + `) + + editor.screenLineForScreenRow(0) + + editor.setCursorBufferPosition([1, 3]) + editor.selectLargerSyntaxNode() + expect(editor.getSelectedText()).toBe('eee') + editor.selectLargerSyntaxNode() + expect(editor.getSelectedText()).toBe('eee.f') + editor.selectLargerSyntaxNode() + expect(editor.getSelectedText()).toBe('eee.f()') + editor.selectLargerSyntaxNode() + expect(editor.getSelectedText()).toBe('{\n eee.f()\n g()\n}') + editor.selectLargerSyntaxNode() + expect(editor.getSelectedText()).toBe('function a (b, c, d) {\n eee.f()\n g()\n}') + + editor.selectSmallerSyntaxNode() + expect(editor.getSelectedText()).toBe('{\n eee.f()\n g()\n}') + editor.selectSmallerSyntaxNode() + expect(editor.getSelectedText()).toBe('eee.f()') + editor.selectSmallerSyntaxNode() + expect(editor.getSelectedText()).toBe('eee.f') + editor.selectSmallerSyntaxNode() + expect(editor.getSelectedText()).toBe('eee') + editor.selectSmallerSyntaxNode() + expect(editor.getSelectedBufferRange()).toEqual([[1, 3], [1, 3]]) + }) + }) }) function getDisplayText (editor) { From a475baf4b5ea3ab6e16dd40c897a56b9746eb5f5 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Tue, 5 Dec 2017 12:39:52 -0800 Subject: [PATCH 15/39] Rework fold API for tree-sitter grammars --- spec/tree-sitter-language-mode-spec.js | 88 +++++++++++++++++---- src/tree-sitter-grammar.js | 7 +- src/tree-sitter-language-mode.js | 103 ++++++++++++++++--------- 3 files changed, 140 insertions(+), 58 deletions(-) diff --git a/spec/tree-sitter-language-mode-spec.js b/spec/tree-sitter-language-mode-spec.js index b05147631..1cc9afc94 100644 --- a/spec/tree-sitter-language-mode-spec.js +++ b/spec/tree-sitter-language-mode-spec.js @@ -76,13 +76,16 @@ describe('TreeSitterLanguageMode', () => { it('can fold nodes that start and end with specified tokens and span multiple lines', () => { const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, { parser: 'tree-sitter-javascript', - scopes: {'program': 'source'}, - folds: { - delimiters: [ - ['{', '}'], - ['(', ')'] - ] - } + folds: [ + { + start: {type: '{', index: 0}, + end: {type: '}', index: -1} + }, + { + start: {type: '(', index: 0}, + end: {type: ')', index: -1} + } + ] }) buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar})) @@ -92,7 +95,7 @@ describe('TreeSitterLanguageMode', () => { getB (c, d, e) { - return this.b + return this.f(g) } } `) @@ -110,7 +113,7 @@ describe('TreeSitterLanguageMode', () => { module.exports = class A { getB (…) { - return this.b + return this.f(g) } } `) @@ -124,16 +127,69 @@ describe('TreeSitterLanguageMode', () => { `) }) + it('can fold nodes that start and end with specified tokens and span multiple lines', () => { + const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, { + parser: 'tree-sitter-javascript', + folds: [ + { + type: 'jsx_element', + start: {index: 0, type: 'jsx_opening_element'}, + end: {index: -1, type: 'jsx_closing_element'} + }, + { + type: 'jsx_self_closing_element', + start: {index: 1}, + end: {type: '/', index: -2} + }, + ] + }) + + buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar})) + buffer.setText(dedent ` + const element1 = + + const element2 = + hello + world + + `) + + editor.screenLineForScreenRow(0) + + expect(editor.isFoldableAtBufferRow(0)).toBe(true) + expect(editor.isFoldableAtBufferRow(1)).toBe(false) + expect(editor.isFoldableAtBufferRow(2)).toBe(false) + expect(editor.isFoldableAtBufferRow(3)).toBe(false) + expect(editor.isFoldableAtBufferRow(4)).toBe(true) + expect(editor.isFoldableAtBufferRow(5)).toBe(false) + + editor.foldBufferRow(0) + expect(getDisplayText(editor)).toBe(dedent ` + const element1 = + + const element2 = + hello + world + + `) + + editor.foldBufferRow(4) + expect(getDisplayText(editor)).toBe(dedent ` + const element1 = + + const element2 = … + `) + }) + it('can fold specified types of multi-line nodes', () => { const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, { parser: 'tree-sitter-javascript', - scopes: {'program': 'source'}, - folds: { - nodes: [ - 'template_string', - 'comment' - ] - } + folds: [ + {type: 'template_string'}, + {type: 'comment'} + ] }) buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar})) diff --git a/src/tree-sitter-grammar.js b/src/tree-sitter-grammar.js index d7d36a0a7..3448d0cd1 100644 --- a/src/tree-sitter-grammar.js +++ b/src/tree-sitter-grammar.js @@ -10,10 +10,7 @@ class TreeSitterGrammar { this.id = params.id this.name = params.name - this.foldConfig = { - delimiters: params.folds && params.folds.delimiters || [], - nodes: new Set(params.folds && params.folds.nodes || []) - } + this.folds = params.folds || [] this.commentStrings = { commentStartString: params.comments && params.comments.start, @@ -21,7 +18,7 @@ class TreeSitterGrammar { } const scopeSelectors = {} - for (const key of Object.keys(params.scopes)) { + for (const key in params.scopes || {}) { scopeSelectors[key] = params.scopes[key] .split('.') .map(s => `syntax--${s}`) diff --git a/src/tree-sitter-language-mode.js b/src/tree-sitter-language-mode.js index 8d4049a51..ff7d6c096 100644 --- a/src/tree-sitter-language-mode.js +++ b/src/tree-sitter-language-mode.js @@ -18,6 +18,7 @@ class TreeSitterLanguageMode { this.document.parse() this.rootScopeDescriptor = new ScopeDescriptor({scopes: [this.grammar.id]}) this.emitter = new Emitter() + this.isFoldableCache = [] } getLanguageId () { @@ -25,6 +26,7 @@ class TreeSitterLanguageMode { } bufferDidChange ({oldRange, newRange, oldText, newText}) { + this.isFoldableCache.length = 0 this.document.edit({ startIndex: this.buffer.characterIndexForPosition(oldRange.start), lengthRemoved: oldText.length, @@ -112,7 +114,10 @@ class TreeSitterLanguageMode { */ isFoldableAtRow (row) { - return this.getFoldableRangeContainingPoint(Point(row, Infinity), 0, true) != null + if (this.isFoldableCache[row] != null) return this.isFoldableCache[row] + const result = this.getFoldableRangeContainingPoint(Point(row, Infinity), 0, true) != null + this.isFoldableCache[row] = result + return result } getFoldableRanges () { @@ -174,48 +179,72 @@ class TreeSitterLanguageMode { } getFoldableRangeForNode (node, existenceOnly) { - const {firstChild} = node - if (firstChild) { - const {lastChild} = node + const {children, type: nodeType} = node + const childCount = children.length + let childTypes - for (let i = 0, n = this.grammar.foldConfig.delimiters.length; i < n; i++) { - const entry = this.grammar.foldConfig.delimiters[i] - if (firstChild.type === entry[0] && lastChild.type === entry[1]) { - if (existenceOnly) return true - let childPrecedingFold = firstChild + for (var i = 0, {length} = this.grammar.folds; i < length; i++) { + const foldEntry = this.grammar.folds[i] - const options = entry[2] - if (options) { - const {children} = node - let childIndexPrecedingFold = options.afterChildCount || 0 - if (options.afterType) { - for (let i = childIndexPrecedingFold, n = children.length; i < n; i++) { - if (children[i].type === options.afterType) { - childIndexPrecedingFold = i - break - } - } - } - childPrecedingFold = children[childIndexPrecedingFold] - } - - let granchildPrecedingFold = childPrecedingFold.lastChild - if (granchildPrecedingFold) { - return Range(granchildPrecedingFold.endPosition, lastChild.startPosition) - } else { - return Range(childPrecedingFold.endPosition, lastChild.startPosition) - } + if (foldEntry.type) { + if (typeof foldEntry.type === 'string') { + if (foldEntry.type !== nodeType) continue + } else { + if (!foldEntry.type.includes(nodeType)) continue + } + } + + let childBeforeFold + const startEntry = foldEntry.start + if (startEntry) { + if (startEntry.index != null) { + childBeforeFold = children[startEntry.index] + if (!childBeforeFold) continue + if (startEntry.type && startEntry.type !== childBeforeFold.type) continue + } else { + if (!childTypes) childTypes = children.map(child => child.type) + let index = childTypes.indexOf(startEntry.type) + if (index === -1) continue + childBeforeFold = children[index] + } + } + + let childAfterFold + const endEntry = foldEntry.end + if (endEntry) { + if (endEntry.index != null) { + const index = endEntry.index < 0 ? childCount + endEntry.index : endEntry.index + childAfterFold = children[index] + if (!childAfterFold) continue + if (endEntry.type && endEntry.type !== childAfterFold.type) continue + } else { + if (!childTypes) childTypes = children.map(child => child.type) + let index = childTypes.lastIndexOf(endEntry.type) + if (index === -1) continue + childAfterFold = children[index] } } - } - if (this.grammar.foldConfig.nodes.has(node.type)) { if (existenceOnly) return true - const start = node.startPosition - const end = node.endPosition - start.column = Infinity - end.column = 0 - return Range(start, end) + + let start, end + if (childBeforeFold) { + start = childBeforeFold.endPosition + } else { + start = new Point(node.startPosition.row, Infinity) + } + if (childAfterFold) { + end = childAfterFold.startPosition + } else { + const {endPosition} = node + if (endPosition.column === 0) { + end = Point(endPosition.row - 1, Infinity) + } else { + end = Point(endPosition.row, 0) + } + } + + return new Range(start, end) } } From f3715779e5d00266147776ffda9b51f6aedbb45f Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Tue, 5 Dec 2017 16:26:24 -0800 Subject: [PATCH 16/39] Support contentRegExp field on grammars, to match more than one line Signed-off-by: Nathan Sobo --- spec/grammar-registry-spec.js | 27 +++++++++++++++++ src/grammar-registry.js | 57 ++++++++++++++++++++--------------- src/tree-sitter-grammar.js | 3 +- 3 files changed, 60 insertions(+), 27 deletions(-) diff --git a/spec/grammar-registry-spec.js b/spec/grammar-registry-spec.js index 4066af24d..7b8f6f1b2 100644 --- a/spec/grammar-registry-spec.js +++ b/spec/grammar-registry-spec.js @@ -1,5 +1,6 @@ const {it, fit, ffit, fffit, beforeEach, afterEach} = require('./async-spec-helpers') +const dedent = require('dedent') const path = require('path') const fs = require('fs-plus') const temp = require('temp').track() @@ -273,6 +274,32 @@ describe('GrammarRegistry', () => { expect(atom.grammars.selectGrammar('/hu.git/config').name).toBe('Null Grammar') }) + describe('when the grammar has a contentRegExp field', () => { + it('favors grammars whose contentRegExp matches a prefix of the file\'s content', () => { + atom.grammars.addGrammar({ + id: 'javascript-1', + fileTypes: ['js'] + }) + atom.grammars.addGrammar({ + id: 'flow-javascript', + contentRegExp: new RegExp('//.*@flow'), + fileTypes: ['js'] + }) + atom.grammars.addGrammar({ + id: 'javascript-2', + fileTypes: ['js'] + }) + + const selectedGrammar = atom.grammars.selectGrammar('test.js', dedent` + // Copyright EvilCorp + // @flow + + module.exports = function () { return 1 + 1 } + `) + expect(selectedGrammar.id).toBe('flow-javascript') + }) + }) + it("uses the filePath's shebang line if the grammar cannot be determined by the extension or basename", async () => { await atom.packages.activatePackage('language-javascript') await atom.packages.activatePackage('language-ruby') diff --git a/src/grammar-registry.js b/src/grammar-registry.js index 6dbb248e7..6722e097b 100644 --- a/src/grammar-registry.js +++ b/src/grammar-registry.js @@ -11,7 +11,6 @@ const fs = require('fs-plus') const {Point, Range} = require('text-buffer') const GRAMMAR_TYPE_BONUS = 1000 -const GRAMMAR_SELECTION_RANGE = Range(Point.ZERO, Point(10, 0)).freeze() const PATH_SPLIT_REGEX = new RegExp('[/.]') const LANGUAGE_ID_MAP = [ @@ -147,7 +146,7 @@ class GrammarRegistry { autoAssignLanguageMode (buffer) { const result = this.selectGrammarWithScore( buffer.getPath(), - buffer.getTextInRange(GRAMMAR_SELECTION_RANGE) + getGrammarSelectionContent(buffer) ) this.languageOverridesByBufferId.delete(buffer.id) this.grammarScoresByBuffer.set(buffer, result.score) @@ -245,26 +244,32 @@ class GrammarRegistry { } grammarMatchesContents (grammar, contents) { - if (contents == null || grammar.firstLineRegex == null) return false + if (contents == null) return false - let escaped = false - let numberOfNewlinesInRegex = 0 - for (let character of grammar.firstLineRegex.source) { - switch (character) { - case '\\': - escaped = !escaped - break - case 'n': - if (escaped) { numberOfNewlinesInRegex++ } - escaped = false - break - default: - escaped = false + if (grammar.contentRegExp) { // TreeSitter grammars + return grammar.contentRegExp.test(contents) + } else if (grammar.firstLineRegex) { // FirstMate grammars + let escaped = false + let numberOfNewlinesInRegex = 0 + for (let character of grammar.firstLineRegex.source) { + switch (character) { + case '\\': + escaped = !escaped + break + case 'n': + if (escaped) { numberOfNewlinesInRegex++ } + escaped = false + break + default: + escaped = false + } } - } - const lines = contents.split('\n') - return grammar.firstLineRegex.testSync(lines.slice(0, numberOfNewlinesInRegex + 1).join('\n')) + const lines = contents.split('\n') + return grammar.firstLineRegex.testSync(lines.slice(0, numberOfNewlinesInRegex + 1).join('\n')) + } else { + return false + } } forEachGrammar (callback) { @@ -338,12 +343,7 @@ class GrammarRegistry { grammar.id === languageOverride)) { buffer.setLanguageMode(this.languageModeForGrammarAndBuffer(grammar, buffer)) } else if (!languageOverride) { - const score = this.getGrammarScore( - grammar, - buffer.getPath(), - buffer.getTextInRange(GRAMMAR_SELECTION_RANGE) - ) - + const score = this.getGrammarScore(grammar, buffer.getPath(), getGrammarSelectionContent(buffer)) const currentScore = this.grammarScoresByBuffer.get(buffer) if (currentScore == null || score > currentScore) { buffer.setLanguageMode(this.languageModeForGrammarAndBuffer(grammar, buffer)) @@ -500,3 +500,10 @@ class GrammarRegistry { } } } + +function getGrammarSelectionContent (buffer) { + return buffer.getTextInRange(Range( + Point(0, 0), + buffer.positionForCharacterIndex(1024) + )) +} diff --git a/src/tree-sitter-grammar.js b/src/tree-sitter-grammar.js index 3448d0cd1..b36505a0b 100644 --- a/src/tree-sitter-grammar.js +++ b/src/tree-sitter-grammar.js @@ -1,7 +1,6 @@ const path = require('path') const SyntaxScopeMap = require('./syntax-scope-map') const Module = require('module') -const {OnigRegExp} = require('oniguruma') module.exports = class TreeSitterGrammar { @@ -9,6 +8,7 @@ class TreeSitterGrammar { this.registry = registry this.id = params.id this.name = params.name + if (params.contentRegExp) this.contentRegExp = new RegExp(params.contentRegExp) this.folds = params.folds || [] @@ -37,7 +37,6 @@ class TreeSitterGrammar { }) this.languageModule = require(languageModulePath) - this.firstLineRegex = params.firstLineMatch && new OnigRegExp(params.firstLineMatch) this.scopesById = new Map() this.idsByScope = {} this.nextScopeId = 256 + 1 From 77fd29647a6f756a668268551fd11bc88600be60 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Tue, 5 Dec 2017 17:01:49 -0800 Subject: [PATCH 17/39] Cache foldability more intelligently Signed-off-by: Nathan Sobo --- src/tree-sitter-language-mode.js | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/tree-sitter-language-mode.js b/src/tree-sitter-language-mode.js index ff7d6c096..166816d0d 100644 --- a/src/tree-sitter-language-mode.js +++ b/src/tree-sitter-language-mode.js @@ -26,7 +26,10 @@ class TreeSitterLanguageMode { } bufferDidChange ({oldRange, newRange, oldText, newText}) { - this.isFoldableCache.length = 0 + const startRow = oldRange.start.row + const oldEndRow = oldRange.end.row + const newEndRow = newRange.end.row + this.isFoldableCache.splice(startRow, oldEndRow - startRow, ...new Array(newEndRow - startRow)) this.document.edit({ startIndex: this.buffer.characterIndexForPosition(oldRange.start), lengthRemoved: oldText.length, @@ -44,7 +47,13 @@ class TreeSitterLanguageMode { buildHighlightIterator () { const invalidatedRanges = this.document.parse() for (let i = 0, n = invalidatedRanges.length; i < n; i++) { - this.emitter.emit('did-change-highlighting', invalidatedRanges[i]) + const range = invalidatedRanges[i] + const startRow = range.start.row + const endRow = range.end.row + for (let row = startRow; row < endRow; row++) { + this.isFoldableCache[row] = undefined + } + this.emitter.emit('did-change-highlighting', range) } return new TreeSitterHighlightIterator(this) } From 815b445d2e78c7f0a04fcd65dd8e924d08ca8883 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Tue, 5 Dec 2017 17:58:39 -0800 Subject: [PATCH 18/39] :arrow_up: language packages --- package.json | 12 ++++++------ src/grammar-registry.js | 7 ++++++- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/package.json b/package.json index 91cf950b4..4c9fa8389 100644 --- a/package.json +++ b/package.json @@ -137,18 +137,18 @@ "welcome": "0.36.6", "whitespace": "0.37.5", "wrap-guide": "0.40.3", - "language-c": "0.58.1", + "language-c": "0.59.0-1", "language-clojure": "0.22.5", "language-coffee-script": "0.49.3", "language-csharp": "0.14.3", "language-css": "0.42.8", "language-gfm": "0.90.2", "language-git": "0.19.1", - "language-go": "0.44.3", + "language-go": "0.45.0-2", "language-html": "0.48.3", "language-hyperlink": "0.16.3", "language-java": "0.27.6", - "language-javascript": "0.128.0-0", + "language-javascript": "0.128.0-1", "language-json": "0.19.1", "language-less": "0.34.1", "language-make": "0.22.3", @@ -157,17 +157,17 @@ "language-perl": "0.38.1", "language-php": "0.42.2", "language-property-list": "0.9.1", - "language-python": "0.45.5", + "language-python": "0.46.0-0", "language-ruby": "0.71.4", "language-ruby-on-rails": "0.25.2", "language-sass": "0.61.3", - "language-shellscript": "0.25.4", + "language-shellscript": "0.26.0-0", "language-source": "0.9.0", "language-sql": "0.25.8", "language-text": "0.7.3", "language-todo": "0.29.3", "language-toml": "0.18.1", - "language-typescript": "0.2.3", + "language-typescript": "0.3.0-0", "language-xml": "0.35.2", "language-yaml": "0.31.1" }, diff --git a/src/grammar-registry.js b/src/grammar-registry.js index 6722e097b..dd11171ba 100644 --- a/src/grammar-registry.js +++ b/src/grammar-registry.js @@ -15,7 +15,12 @@ const PATH_SPLIT_REGEX = new RegExp('[/.]') const LANGUAGE_ID_MAP = [ ['source.js', 'javascript'], - ['source.ts', 'typescript'] + ['source.ts', 'typescript'], + ['source.c', 'c'], + ['source.cpp', 'cpp'], + ['source.go', 'go'], + ['source.python', 'python'], + ['source.sh', 'bash'] ] // Extended: This class holds the grammars used for tokenizing. From 3f775b550510ec2483c3fc8220dd5acf54f87449 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 6 Dec 2017 11:09:44 -0800 Subject: [PATCH 19/39] Fix folding of internal nodes when fold end isn't specified --- spec/tree-sitter-language-mode-spec.js | 157 ++++++++++++++++++++++--- src/tree-sitter-language-mode.js | 44 ++++--- 2 files changed, 163 insertions(+), 38 deletions(-) diff --git a/spec/tree-sitter-language-mode-spec.js b/spec/tree-sitter-language-mode-spec.js index 1cc9afc94..426291e5f 100644 --- a/spec/tree-sitter-language-mode-spec.js +++ b/spec/tree-sitter-language-mode-spec.js @@ -6,6 +6,8 @@ const TextEditor = require('../src/text-editor') const TreeSitterGrammar = require('../src/tree-sitter-grammar') const TreeSitterLanguageMode = require('../src/tree-sitter-language-mode') +const cGrammarPath = require.resolve('language-c/grammars/tree-sitter-c.cson') +const pythonGrammarPath = require.resolve('language-python/grammars/tree-sitter-python.cson') const jsGrammarPath = require.resolve('language-javascript/grammars/tree-sitter-javascript.cson') describe('TreeSitterLanguageMode', () => { @@ -73,7 +75,7 @@ describe('TreeSitterLanguageMode', () => { editor.displayLayer.reset({foldCharacter: '…'}) }) - it('can fold nodes that start and end with specified tokens and span multiple lines', () => { + it('can fold nodes that start and end with specified tokens', () => { const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, { parser: 'tree-sitter-javascript', folds: [ @@ -107,6 +109,7 @@ describe('TreeSitterLanguageMode', () => { expect(editor.isFoldableAtBufferRow(2)).toBe(true) expect(editor.isFoldableAtBufferRow(3)).toBe(false) expect(editor.isFoldableAtBufferRow(4)).toBe(true) + expect(editor.isFoldableAtBufferRow(5)).toBe(false) editor.foldBufferRow(2) expect(getDisplayText(editor)).toBe(dedent ` @@ -127,20 +130,24 @@ describe('TreeSitterLanguageMode', () => { `) }) - it('can fold nodes that start and end with specified tokens and span multiple lines', () => { + it('can fold nodes of specified types', () => { const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, { parser: 'tree-sitter-javascript', folds: [ + // Start the fold after the first child (the opening tag) and end it at the last child + // (the closing tag). { type: 'jsx_element', - start: {index: 0, type: 'jsx_opening_element'}, - end: {index: -1, type: 'jsx_closing_element'} + start: {index: 0}, + end: {index: -1} }, + + // End the fold at the *second* to last child of the self-closing tag: the `/`. { type: 'jsx_self_closing_element', start: {index: 1}, - end: {type: '/', index: -2} - }, + end: {index: -2} + } ] }) @@ -183,11 +190,12 @@ describe('TreeSitterLanguageMode', () => { `) }) - it('can fold specified types of multi-line nodes', () => { + it('can fold entire nodes when no start or end parameters are specified', () => { const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, { parser: 'tree-sitter-javascript', folds: [ - {type: 'template_string'}, + // By default, for a node with no children, folds are started at the *end* of the first + // line of a node, and ended at the *beginning* of the last line. {type: 'comment'} ] }) @@ -197,9 +205,9 @@ describe('TreeSitterLanguageMode', () => { /** * Important */ - const x = \`one - two - three\` + const x = 1 /* + Also important + */ `) editor.screenLineForScreenRow(0) @@ -213,17 +221,136 @@ describe('TreeSitterLanguageMode', () => { editor.foldBufferRow(0) expect(getDisplayText(editor)).toBe(dedent ` /**… */ - const x = \`one - two - three\` + const x = 1 /* + Also important + */ `) editor.foldBufferRow(3) expect(getDisplayText(editor)).toBe(dedent ` /**… */ - const x = \`one… three\` + const x = 1 /*…*/ `) }) + + it('tries each folding strategy for a given node in the order specified', () => { + const grammar = new TreeSitterGrammar(atom.grammars, cGrammarPath, { + parser: 'tree-sitter-c', + folds: [ + // If the #ifdef has an `#else` clause, then end the fold there. + { + type: 'preproc_ifdef', + start: {index: 1}, + end: {type: 'preproc_else'} + }, + + // Otherwise, end the fold at the last child - the `#endif`. + { + type: 'preproc_ifdef', + start: {index: 1}, + end: {index: -1} + }, + + // When folding an `#else` clause, the fold extends to the end of the clause. + { + type: 'preproc_else', + start: {index: 0} + } + ] + }) + + buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar})) + + buffer.setText(dedent ` + #ifndef FOO_H_ + #define FOO_H_ + + #ifdef _WIN32 + + #include + const char *path_separator = "\\"; + + #else + + #include + const char *path_separator = "/"; + + #endif + + #endif + `) + + editor.screenLineForScreenRow(0) + + editor.foldBufferRow(3) + expect(getDisplayText(editor)).toBe(dedent ` + #ifndef FOO_H_ + #define FOO_H_ + + #ifdef _WIN32…#else + + #include + const char *path_separator = "/"; + + #endif + + #endif + `) + + editor.foldBufferRow(8) + expect(getDisplayText(editor)).toBe(dedent ` + #ifndef FOO_H_ + #define FOO_H_ + + #ifdef _WIN32…#else… + + #endif + + #endif + `) + + editor.foldBufferRow(0) + expect(getDisplayText(editor)).toBe(dedent ` + #ifndef FOO_H_…#endif + `) + }) + + describe('when folding a node that ends with a line break', () => { + it('ends the fold at the end of the previous line', () => { + const grammar = new TreeSitterGrammar(atom.grammars, pythonGrammarPath, { + parser: 'tree-sitter-python', + folds: [ + { + type: 'function_definition', + start: {type: ':'} + } + ] + }) + + buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar})) + + buffer.setText(dedent ` + def ab(): + print 'a' + print 'b' + + def cd(): + print 'c' + print 'd' + `) + + editor.screenLineForScreenRow(0) + + editor.foldBufferRow(0) + expect(getDisplayText(editor)).toBe(dedent ` + def ab():… + + def cd(): + print 'c' + print 'd' + `) + }) + }) }) describe('TextEditor.selectLargerSyntaxNode and .selectSmallerSyntaxNode', () => { diff --git a/src/tree-sitter-language-mode.js b/src/tree-sitter-language-mode.js index 166816d0d..f47d89db7 100644 --- a/src/tree-sitter-language-mode.js +++ b/src/tree-sitter-language-mode.js @@ -203,57 +203,55 @@ class TreeSitterLanguageMode { } } - let childBeforeFold + let foldStart const startEntry = foldEntry.start if (startEntry) { if (startEntry.index != null) { - childBeforeFold = children[startEntry.index] - if (!childBeforeFold) continue - if (startEntry.type && startEntry.type !== childBeforeFold.type) continue + const child = children[startEntry.index] + if (!child || (startEntry.type && startEntry.type !== child.type)) continue + foldStart = child.endPosition } else { if (!childTypes) childTypes = children.map(child => child.type) - let index = childTypes.indexOf(startEntry.type) + const index = childTypes.indexOf(startEntry.type) if (index === -1) continue - childBeforeFold = children[index] + foldStart = children[index].endPosition } } - let childAfterFold + let foldEnd const endEntry = foldEntry.end if (endEntry) { if (endEntry.index != null) { const index = endEntry.index < 0 ? childCount + endEntry.index : endEntry.index - childAfterFold = children[index] - if (!childAfterFold) continue - if (endEntry.type && endEntry.type !== childAfterFold.type) continue + const child = children[index] + if (!child || (endEntry.type && endEntry.type !== child.type)) continue + foldEnd = child.startPosition } else { if (!childTypes) childTypes = children.map(child => child.type) - let index = childTypes.lastIndexOf(endEntry.type) + const index = childTypes.lastIndexOf(endEntry.type) if (index === -1) continue - childAfterFold = children[index] + foldEnd = children[index].startPosition } } if (existenceOnly) return true - let start, end - if (childBeforeFold) { - start = childBeforeFold.endPosition - } else { - start = new Point(node.startPosition.row, Infinity) + if (!foldStart) { + foldStart = new Point(node.startPosition.row, Infinity) } - if (childAfterFold) { - end = childAfterFold.startPosition - } else { + + if (!foldEnd) { const {endPosition} = node if (endPosition.column === 0) { - end = Point(endPosition.row - 1, Infinity) + foldEnd = Point(endPosition.row - 1, Infinity) + } else if (childCount > 0) { + foldEnd = endPosition } else { - end = Point(endPosition.row, 0) + foldEnd = Point(endPosition.row, 0) } } - return new Range(start, end) + return new Range(foldStart, foldEnd) } } From 4c6abd3b7a8fde0eacda3f620f8a48ec8bc58058 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 6 Dec 2017 14:16:25 -0800 Subject: [PATCH 20/39] :arrow_up: language-javascript, language-typescript --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 4c9fa8389..fa50740de 100644 --- a/package.json +++ b/package.json @@ -148,7 +148,7 @@ "language-html": "0.48.3", "language-hyperlink": "0.16.3", "language-java": "0.27.6", - "language-javascript": "0.128.0-1", + "language-javascript": "0.128.0-2", "language-json": "0.19.1", "language-less": "0.34.1", "language-make": "0.22.3", @@ -167,7 +167,7 @@ "language-text": "0.7.3", "language-todo": "0.29.3", "language-toml": "0.18.1", - "language-typescript": "0.3.0-0", + "language-typescript": "0.3.0-1", "language-xml": "0.35.2", "language-yaml": "0.31.1" }, From 264de98d927aba90bae0323b65e0631ea8da1d6d Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 7 Dec 2017 11:54:03 -0800 Subject: [PATCH 21/39] :arrow_up: tree-sitter --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 5fbf1dbf7..339c5313d 100644 --- a/package.json +++ b/package.json @@ -71,7 +71,7 @@ "sinon": "1.17.4", "temp": "^0.8.3", "text-buffer": "13.9.2", - "tree-sitter": "0.7.4", + "tree-sitter": "0.7.5", "typescript-simple": "1.0.0", "underscore-plus": "^1.6.6", "winreg": "^1.2.1", From 136dc86584a04f42b69b01685af8ec8c8403ca88 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 7 Dec 2017 15:29:11 -0800 Subject: [PATCH 22/39] Leave muli-character fold end tokens on their own line Signed-off-by: Nathan Sobo --- spec/tree-sitter-language-mode-spec.js | 33 ++++++++++++++++++++++---- src/tree-sitter-language-mode.js | 16 +++++++++---- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/spec/tree-sitter-language-mode-spec.js b/spec/tree-sitter-language-mode-spec.js index 426291e5f..0eeeb8b93 100644 --- a/spec/tree-sitter-language-mode-spec.js +++ b/spec/tree-sitter-language-mode-spec.js @@ -186,7 +186,8 @@ describe('TreeSitterLanguageMode', () => { expect(getDisplayText(editor)).toBe(dedent ` const element1 = - const element2 = … + const element2 = … + `) }) @@ -239,10 +240,15 @@ describe('TreeSitterLanguageMode', () => { folds: [ // If the #ifdef has an `#else` clause, then end the fold there. { - type: 'preproc_ifdef', + type: ['preproc_ifdef', 'preproc_elif'], start: {index: 1}, end: {type: 'preproc_else'} }, + { + type: ['preproc_ifdef', 'preproc_elif'], + start: {index: 1}, + end: {type: 'preproc_elif'} + }, // Otherwise, end the fold at the last child - the `#endif`. { @@ -270,6 +276,11 @@ describe('TreeSitterLanguageMode', () => { #include const char *path_separator = "\\"; + #elif defined MACOS + + #include + const char *path_separator = "/"; + #else #include @@ -287,7 +298,13 @@ describe('TreeSitterLanguageMode', () => { #ifndef FOO_H_ #define FOO_H_ - #ifdef _WIN32…#else + #ifdef _WIN32… + #elif defined MACOS + + #include + const char *path_separator = "/"; + + #else #include const char *path_separator = "/"; @@ -302,7 +319,12 @@ describe('TreeSitterLanguageMode', () => { #ifndef FOO_H_ #define FOO_H_ - #ifdef _WIN32…#else… + #ifdef _WIN32… + #elif defined MACOS… + #else + + #include + const char *path_separator = "/"; #endif @@ -311,7 +333,8 @@ describe('TreeSitterLanguageMode', () => { editor.foldBufferRow(0) expect(getDisplayText(editor)).toBe(dedent ` - #ifndef FOO_H_…#endif + #ifndef FOO_H_… + #endif `) }) diff --git a/src/tree-sitter-language-mode.js b/src/tree-sitter-language-mode.js index f47d89db7..a76043638 100644 --- a/src/tree-sitter-language-mode.js +++ b/src/tree-sitter-language-mode.js @@ -221,16 +221,22 @@ class TreeSitterLanguageMode { let foldEnd const endEntry = foldEntry.end if (endEntry) { + let foldEndNode if (endEntry.index != null) { const index = endEntry.index < 0 ? childCount + endEntry.index : endEntry.index - const child = children[index] - if (!child || (endEntry.type && endEntry.type !== child.type)) continue - foldEnd = child.startPosition + foldEndNode = children[index] + if (!foldEndNode || (endEntry.type && endEntry.type !== foldEndNode.type)) continue } else { - if (!childTypes) childTypes = children.map(child => child.type) + if (!childTypes) childTypes = children.map(foldEndNode => foldEndNode.type) const index = childTypes.lastIndexOf(endEntry.type) if (index === -1) continue - foldEnd = children[index].startPosition + foldEndNode = children[index] + } + + if (foldEndNode.endIndex - foldEndNode.startIndex > 1 && foldEndNode.startPosition.row > foldStart.row) { + foldEnd = new Point(foldEndNode.startPosition.row - 1, Infinity) + } else { + foldEnd = foldEndNode.startPosition } } From f712de65d0c2ea2a6c8cc2fefd2efdde8d5910a3 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 7 Dec 2017 15:30:48 -0800 Subject: [PATCH 23/39] Fix nesting level calculation for children of partially-folded nodes Signed-off-by: Nathan Sobo --- spec/tree-sitter-language-mode-spec.js | 14 ++++++++++++++ src/text-editor.js | 2 +- src/tree-sitter-language-mode.js | 22 ++++++++++++---------- 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/spec/tree-sitter-language-mode-spec.js b/spec/tree-sitter-language-mode-spec.js index 0eeeb8b93..fe9ec239b 100644 --- a/spec/tree-sitter-language-mode-spec.js +++ b/spec/tree-sitter-language-mode-spec.js @@ -336,6 +336,20 @@ describe('TreeSitterLanguageMode', () => { #ifndef FOO_H_… #endif `) + + editor.foldAllAtIndentLevel(1) + expect(getDisplayText(editor)).toBe(dedent ` + #ifndef FOO_H_ + #define FOO_H_ + + #ifdef _WIN32… + #elif defined MACOS… + #else… + + #endif + + #endif + `) }) describe('when folding a node that ends with a line break', () => { diff --git a/src/text-editor.js b/src/text-editor.js index e24476b2d..08d07aa71 100644 --- a/src/text-editor.js +++ b/src/text-editor.js @@ -3891,7 +3891,7 @@ class TextEditor { // Extended: Fold all foldable lines at the given indent level. // - // * `level` A {Number}. + // * `level` A {Number} starting at 0. foldAllAtIndentLevel (level) { const languageMode = this.buffer.getLanguageMode() const foldableRanges = ( diff --git a/src/tree-sitter-language-mode.js b/src/tree-sitter-language-mode.js index a76043638..6eec047c3 100644 --- a/src/tree-sitter-language-mode.js +++ b/src/tree-sitter-language-mode.js @@ -138,10 +138,7 @@ class TreeSitterLanguageMode { let stack = [{node: this.document.rootNode, level: 0}] while (stack.length > 0) { const {node, level} = stack.pop() - const startRow = node.startPosition.row - const endRow = node.endPosition.row - let childLevel = level const range = this.getFoldableRangeForNode(node) if (range) { if (goalLevel == null || level === goalLevel) { @@ -155,18 +152,23 @@ class TreeSitterLanguageMode { } if (!updatedExistingRange) result.push(range) } - childLevel++ } + const parentStartRow = node.startPosition.row + const parentEndRow = node.endPosition.row for (let children = node.namedChildren, i = 0, {length} = children; i < length; i++) { const child = children[i] - const childStartRow = child.startPosition.row - const childEndRow = child.endPosition.row - if (childEndRow > childStartRow) { - if (childStartRow === startRow && childEndRow === endRow) { + const {startPosition: childStart, endPosition: childEnd} = child + if (childEnd.row > childStart.row) { + if (childStart.row === parentStartRow && childEnd.row === parentEndRow) { stack.push({node: child, level: level}) - } else if (childLevel <= goalLevel || goalLevel == null) { - stack.push({node: child, level: childLevel}) + } else { + const childLevel = range.containsPoint(childStart) && range.containsPoint(childEnd) + ? level + 1 + : level + if (childLevel <= goalLevel || goalLevel == null) { + stack.push({node: child, level: childLevel}) + } } } } From a7a53f4158cbd302210a10b57617e7e084539776 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 7 Dec 2017 17:08:47 -0800 Subject: [PATCH 24/39] Allow multiple child types to be specified as fold start or end --- spec/tree-sitter-language-mode-spec.js | 7 +------ src/tree-sitter-language-mode.js | 11 ++++++++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/spec/tree-sitter-language-mode-spec.js b/spec/tree-sitter-language-mode-spec.js index fe9ec239b..5ecc73308 100644 --- a/spec/tree-sitter-language-mode-spec.js +++ b/spec/tree-sitter-language-mode-spec.js @@ -242,12 +242,7 @@ describe('TreeSitterLanguageMode', () => { { type: ['preproc_ifdef', 'preproc_elif'], start: {index: 1}, - end: {type: 'preproc_else'} - }, - { - type: ['preproc_ifdef', 'preproc_elif'], - start: {index: 1}, - end: {type: 'preproc_elif'} + end: {type: ['preproc_else', 'preproc_elif']} }, // Otherwise, end the fold at the last child - the `#endif`. diff --git a/src/tree-sitter-language-mode.js b/src/tree-sitter-language-mode.js index 6eec047c3..9f88a71ec 100644 --- a/src/tree-sitter-language-mode.js +++ b/src/tree-sitter-language-mode.js @@ -148,6 +148,7 @@ class TreeSitterLanguageMode { result[i].end.row === range.end.row) { result[i] = range updatedExistingRange = true + break } } if (!updatedExistingRange) result.push(range) @@ -163,7 +164,7 @@ class TreeSitterLanguageMode { if (childStart.row === parentStartRow && childEnd.row === parentEndRow) { stack.push({node: child, level: level}) } else { - const childLevel = range.containsPoint(childStart) && range.containsPoint(childEnd) + const childLevel = range && range.containsPoint(childStart) && range.containsPoint(childEnd) ? level + 1 : level if (childLevel <= goalLevel || goalLevel == null) { @@ -214,7 +215,9 @@ class TreeSitterLanguageMode { foldStart = child.endPosition } else { if (!childTypes) childTypes = children.map(child => child.type) - const index = childTypes.indexOf(startEntry.type) + const index = typeof startEntry.type === 'string' + ? childTypes.indexOf(startEntry.type) + : childTypes.findIndex(type => startEntry.type.includes(type)) if (index === -1) continue foldStart = children[index].endPosition } @@ -230,7 +233,9 @@ class TreeSitterLanguageMode { if (!foldEndNode || (endEntry.type && endEntry.type !== foldEndNode.type)) continue } else { if (!childTypes) childTypes = children.map(foldEndNode => foldEndNode.type) - const index = childTypes.lastIndexOf(endEntry.type) + const index = typeof endEntry.type === 'string' + ? childTypes.indexOf(endEntry.type) + : childTypes.findIndex(type => endEntry.type.includes(type)) if (index === -1) continue foldEndNode = children[index] } From 3d11c1726428766bf9e5b5ec292125950d123f72 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 7 Dec 2017 17:42:52 -0800 Subject: [PATCH 25/39] Fix exception in getFoldableRangeForNode --- src/tree-sitter-language-mode.js | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/tree-sitter-language-mode.js b/src/tree-sitter-language-mode.js index 9f88a71ec..33656cf35 100644 --- a/src/tree-sitter-language-mode.js +++ b/src/tree-sitter-language-mode.js @@ -221,6 +221,8 @@ class TreeSitterLanguageMode { if (index === -1) continue foldStart = children[index].endPosition } + } else { + foldStart = new Point(node.startPosition.row, Infinity) } let foldEnd @@ -245,15 +247,7 @@ class TreeSitterLanguageMode { } else { foldEnd = foldEndNode.startPosition } - } - - if (existenceOnly) return true - - if (!foldStart) { - foldStart = new Point(node.startPosition.row, Infinity) - } - - if (!foldEnd) { + } else { const {endPosition} = node if (endPosition.column === 0) { foldEnd = Point(endPosition.row - 1, Infinity) @@ -264,7 +258,7 @@ class TreeSitterLanguageMode { } } - return new Range(foldStart, foldEnd) + return existenceOnly ? true : new Range(foldStart, foldEnd) } } From e09ee1c1fa8ac10dbec10a0fc47253bb51c7bbd7 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 15 Dec 2017 09:44:45 -0800 Subject: [PATCH 26/39] Fix error in TreeSitterHighlightIterator.seek --- spec/tree-sitter-language-mode-spec.js | 63 ++++++++++++++++++++------ src/tree-sitter-language-mode.js | 2 +- 2 files changed, 51 insertions(+), 14 deletions(-) diff --git a/spec/tree-sitter-language-mode-spec.js b/spec/tree-sitter-language-mode-spec.js index 5ecc73308..91070710f 100644 --- a/spec/tree-sitter-language-mode-spec.js +++ b/spec/tree-sitter-language-mode-spec.js @@ -32,7 +32,7 @@ describe('TreeSitterLanguageMode', () => { buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar})) buffer.setText('aa.bbb = cc(d.eee());') - expectTokensToEqual(editor, [ + expectTokensToEqual(editor, [[ {text: 'aa.', scopes: ['source']}, {text: 'bbb', scopes: ['source', 'property']}, {text: ' = ', scopes: ['source']}, @@ -40,7 +40,7 @@ describe('TreeSitterLanguageMode', () => { {text: '(d.', scopes: ['source']}, {text: 'eee', scopes: ['source', 'method']}, {text: '());', scopes: ['source']} - ]) + ]]) }) it('can start or end multiple scopes at the same position', () => { @@ -58,7 +58,7 @@ describe('TreeSitterLanguageMode', () => { buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar})) buffer.setText('a = bb.ccc();') - expectTokensToEqual(editor, [ + expectTokensToEqual(editor, [[ {text: 'a', scopes: ['source', 'variable']}, {text: ' = ', scopes: ['source']}, {text: 'bb', scopes: ['source', 'call', 'member', 'variable']}, @@ -66,6 +66,31 @@ describe('TreeSitterLanguageMode', () => { {text: '(', scopes: ['source', 'call', 'open-paren']}, {text: ')', scopes: ['source', 'call', 'close-paren']}, {text: ';', scopes: ['source']} + ]]) + }) + + it('can resume highlighting on a line that starts with whitespace', () => { + const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, { + parser: 'tree-sitter-javascript', + scopes: { + 'call_expression > member_expression > property_identifier': 'function', + 'property_identifier': 'member', + 'identifier': 'variable' + } + }) + + buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar})) + buffer.setText('a\n .b();') + expectTokensToEqual(editor, [ + [ + {text: 'a', scopes: ['variable']}, + ], + [ + {text: ' ', scopes: ['whitespace']}, + {text: '.', scopes: []}, + {text: 'b', scopes: ['function']}, + {text: '();', scopes: []} + ] ]) }) }) @@ -432,22 +457,34 @@ function getDisplayText (editor) { return editor.displayLayer.getText() } -function expectTokensToEqual (editor, expectedTokens) { - const tokens = [] - for (let row = 0, lastRow = editor.getLastScreenRow(); row <= lastRow; row++) { - tokens.push( - ...editor.tokensForScreenRow(row).map(({text, scopes}) => ({ +function expectTokensToEqual (editor, expectedTokenLines) { + const lastRow = editor.getLastScreenRow() + + // Assert that the correct tokens are returned regardless of which row + // the highlighting iterator starts on. + for (let startRow = 0; startRow <= lastRow; startRow++) { + editor.displayLayer.clearSpatialIndex() + editor.displayLayer.getScreenLines(startRow, Infinity) + + const tokenLines = [] + for (let row = startRow; row <= lastRow; row++) { + tokenLines[row] = editor.tokensForScreenRow(row).map(({text, scopes}) => ({ text, scopes: scopes.map(scope => scope .split(' ') .map(className => className.slice('syntax--'.length)) .join(' ')) })) - ) - } + } - expect(tokens.length).toEqual(expectedTokens.length) - for (let i = 0; i < tokens.length; i++) { - expect(tokens[i]).toEqual(expectedTokens[i], `Token ${i}`) + for (let row = startRow; row <= lastRow; row++) { + const tokenLine = tokenLines[row] + const expectedTokenLine = expectedTokenLines[row] + + expect(tokenLine.length).toEqual(expectedTokenLine.length) + for (let i = 0; i < tokenLine.length; i++) { + expect(tokenLine[i]).toEqual(expectedTokenLine[i], `Token ${i}, startRow: ${startRow}`) + } + } } } diff --git a/src/tree-sitter-language-mode.js b/src/tree-sitter-language-mode.js index 33656cf35..5cd725108 100644 --- a/src/tree-sitter-language-mode.js +++ b/src/tree-sitter-language-mode.js @@ -349,9 +349,9 @@ class TreeSitterHighlightIterator { do { this.currentNode = node this.currentChildIndex = childIndex + if (!nodeContainsTarget) break this.containingNodeTypes.push(node.type) this.containingNodeChildIndices.push(childIndex) - if (!nodeContainsTarget) break const scopeName = this.currentScopeName() if (scopeName) { From 8efccf822103f6f420d1325fae4fd3e8f24e968d Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 15 Dec 2017 16:55:18 -0800 Subject: [PATCH 27/39] :arrow_up: language packages --- package.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index a041e606f..bec57b304 100644 --- a/package.json +++ b/package.json @@ -137,18 +137,18 @@ "welcome": "0.36.6", "whitespace": "0.37.5", "wrap-guide": "0.40.3", - "language-c": "0.59.0-1", + "language-c": "0.59.0-2", "language-clojure": "0.22.5", "language-coffee-script": "0.49.3", "language-csharp": "0.14.3", "language-css": "0.42.8", "language-gfm": "0.90.3", "language-git": "0.19.1", - "language-go": "0.45.0-2", + "language-go": "0.45.0-3", "language-html": "0.48.4", "language-hyperlink": "0.16.3", "language-java": "0.27.6", - "language-javascript": "0.128.0-2", + "language-javascript": "0.128.0-3", "language-json": "0.19.1", "language-less": "0.34.1", "language-make": "0.22.3", @@ -157,17 +157,17 @@ "language-perl": "0.38.1", "language-php": "0.43.0", "language-property-list": "0.9.1", - "language-python": "0.46.0-0", + "language-python": "0.46.0-1", "language-ruby": "0.71.4", "language-ruby-on-rails": "0.25.3", "language-sass": "0.61.3", - "language-shellscript": "0.26.0-0", + "language-shellscript": "0.26.0-1", "language-source": "0.9.0", "language-sql": "0.25.9", "language-text": "0.7.3", "language-todo": "0.29.3", "language-toml": "0.18.1", - "language-typescript": "0.3.0-1", + "language-typescript": "0.3.0-2", "language-xml": "0.35.2", "language-yaml": "0.31.1" }, From 4adfba47cca5c4aef147918f33a6b2ffb51e6a4a Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 15 Dec 2017 16:57:36 -0800 Subject: [PATCH 28/39] Support legacyScopeName field on tree-sitter grammars * Use the field for mapping scope names in GrammarRegistry.grammarForId * Use the field for adapting legacy scoped settings to work with tree-sitter parsers Signed-off-by: Nathan Sobo --- spec/config-spec.coffee | 23 ++++++++++++++++++ spec/grammar-registry-spec.js | 6 +++++ src/config.coffee | 44 ++++++++++++++++++++++++++++++++--- src/grammar-registry.js | 38 ++++++++++++++++-------------- src/scope-descriptor.coffee | 16 +++++++++---- src/tree-sitter-grammar.js | 1 + 6 files changed, 103 insertions(+), 25 deletions(-) diff --git a/spec/config-spec.coffee b/spec/config-spec.coffee index bcf50c268..090bc7a29 100644 --- a/spec/config-spec.coffee +++ b/spec/config-spec.coffee @@ -106,6 +106,15 @@ describe "Config", -> atom.config.set("foo.bar.baz", 1, scopeSelector: ".source.coffee", source: "some-package") expect(atom.config.get("foo.bar.baz", scope: [".source.coffee"])).toBe 100 + describe "when the first component of the scope descriptor matches a legacy scope alias", -> + it "falls back to properties defined for the legacy scope if no value is found for the original scope descriptor", -> + atom.config.addLegacyScopeAlias('javascript', '.source.js') + atom.config.set('foo', 100, scopeSelector: '.source.js') + atom.config.set('foo', 200, scopeSelector: 'javascript for_statement') + + expect(atom.config.get('foo', scope: ['javascript', 'for_statement', 'identifier'])).toBe(200) + expect(atom.config.get('foo', scope: ['javascript', 'function', 'identifier'])).toBe(100) + describe ".getAll(keyPath, {scope, sources, excludeSources})", -> it "reads all of the values for a given key-path", -> expect(atom.config.set("foo", 41)).toBe true @@ -130,6 +139,20 @@ describe "Config", -> {scopeSelector: '*', value: 40} ] + describe "when the first component of the scope descriptor matches a legacy scope alias", -> + it "includes the values defined for the legacy scope", -> + atom.config.addLegacyScopeAlias('javascript', '.source.js') + + expect(atom.config.set('foo', 41)).toBe true + expect(atom.config.set('foo', 42, scopeSelector: 'javascript')).toBe true + expect(atom.config.set('foo', 43, scopeSelector: '.source.js')).toBe true + + expect(atom.config.getAll('foo', scope: ['javascript'])).toEqual([ + {scopeSelector: 'javascript', value: 42}, + {scopeSelector: '.js.source', value: 43}, + {scopeSelector: '*', value: 41} + ]) + describe ".set(keyPath, value, {source, scopeSelector})", -> it "allows a key path's value to be written", -> expect(atom.config.set("foo.bar.baz", 42)).toBe true diff --git a/spec/grammar-registry-spec.js b/spec/grammar-registry-spec.js index 7b8f6f1b2..e6d815f8d 100644 --- a/spec/grammar-registry-spec.js +++ b/spec/grammar-registry-spec.js @@ -61,6 +61,9 @@ describe('GrammarRegistry', () => { const grammar = grammarRegistry.grammarForId('javascript') expect(grammar instanceof FirstMate.Grammar).toBe(true) expect(grammar.scopeName).toBe('source.js') + + grammarRegistry.removeGrammar(grammar) + expect(grammarRegistry.grammarForId('javascript')).toBe(undefined) }) it('converts the language id to a tree-sitter language id when `core.useTreeSitterParsers` is true', () => { @@ -72,6 +75,9 @@ describe('GrammarRegistry', () => { const grammar = grammarRegistry.grammarForId('source.js') expect(grammar instanceof TreeSitterGrammar).toBe(true) expect(grammar.id).toBe('javascript') + + grammarRegistry.removeGrammar(grammar) + expect(grammarRegistry.grammarForId('source.js') instanceof FirstMate.Grammar).toBe(true) }) }) diff --git a/src/config.coffee b/src/config.coffee index b8bf8a76f..84e726700 100644 --- a/src/config.coffee +++ b/src/config.coffee @@ -423,6 +423,7 @@ class Config @configFileHasErrors = false @transactDepth = 0 @pendingOperations = [] + @legacyScopeAliases = {} @requestLoad = _.debounce => @loadUserConfig() @@ -599,11 +600,22 @@ class Config # * `value` The value for the key-path getAll: (keyPath, options) -> {scope} = options if options? - result = [] if scope? scopeDescriptor = ScopeDescriptor.fromObject(scope) - result = result.concat @scopedSettingsStore.getAll(scopeDescriptor.getScopeChain(), keyPath, options) + result = @scopedSettingsStore.getAll( + scopeDescriptor.getScopeChain(), + keyPath, + options + ) + if legacyScopeDescriptor = @getLegacyScopeDescriptor(scopeDescriptor) + result.push(@scopedSettingsStore.getAll( + legacyScopeDescriptor.getScopeChain(), + keyPath, + options + )...) + else + result = [] if globalValue = @getRawValue(keyPath, options) result.push(scopeSelector: '*', value: globalValue) @@ -762,6 +774,12 @@ class Config finally @endTransaction() + addLegacyScopeAlias: (languageId, legacyScopeName) -> + @legacyScopeAliases[languageId] = legacyScopeName + + removeLegacyScopeAlias: (languageId) -> + delete @legacyScopeAliases[languageId] + ### Section: Internal methods used by core ### @@ -1145,7 +1163,20 @@ class Config getRawScopedValue: (scopeDescriptor, keyPath, options) -> scopeDescriptor = ScopeDescriptor.fromObject(scopeDescriptor) - @scopedSettingsStore.getPropertyValue(scopeDescriptor.getScopeChain(), keyPath, options) + result = @scopedSettingsStore.getPropertyValue( + scopeDescriptor.getScopeChain(), + keyPath, + options + ) + + if result? + result + else if legacyScopeDescriptor = @getLegacyScopeDescriptor(scopeDescriptor) + @scopedSettingsStore.getPropertyValue( + legacyScopeDescriptor.getScopeChain(), + keyPath, + options + ) observeScopedKeyPath: (scope, keyPath, callback) -> callback(@get(keyPath, {scope})) @@ -1160,6 +1191,13 @@ class Config oldValue = newValue callback(event) + getLegacyScopeDescriptor: (scopeDescriptor) -> + legacyAlias = @legacyScopeAliases[scopeDescriptor.scopes[0]] + if legacyAlias + scopes = scopeDescriptor.scopes.slice() + scopes[0] = legacyAlias + new ScopeDescriptor({scopes}) + # Base schema enforcers. These will coerce raw input into the specified type, # and will throw an error when the value cannot be coerced. Throwing the error # will indicate that the value should not be set. diff --git a/src/grammar-registry.js b/src/grammar-registry.js index dd11171ba..b2c4129f7 100644 --- a/src/grammar-registry.js +++ b/src/grammar-registry.js @@ -13,16 +13,6 @@ const {Point, Range} = require('text-buffer') const GRAMMAR_TYPE_BONUS = 1000 const PATH_SPLIT_REGEX = new RegExp('[/.]') -const LANGUAGE_ID_MAP = [ - ['source.js', 'javascript'], - ['source.ts', 'typescript'], - ['source.c', 'c'], - ['source.cpp', 'cpp'], - ['source.go', 'go'], - ['source.python', 'python'], - ['source.sh', 'bash'] -] - // Extended: This class holds the grammars used for tokenizing. // // An instance of this class is always available as the `atom.grammars` global. @@ -42,6 +32,8 @@ class GrammarRegistry { this.subscriptions = new CompositeDisposable() this.languageOverridesByBufferId = new Map() this.grammarScoresByBuffer = new Map() + this.textMateScopeNamesByTreeSitterLanguageId = new Map() + this.treeSitterLanguageIdsByTextMateScopeName = new Map() const grammarAddedOrUpdated = this.grammarAddedOrUpdated.bind(this) this.textmateRegistry.onDidAddGrammar(grammarAddedOrUpdated) @@ -116,7 +108,7 @@ class GrammarRegistry { // Extended: Force a {TextBuffer} to use a different grammar than the // one that would otherwise be selected for it. // - // * `buffer` The {TextBuffer} whose gramamr will be set. + // * `buffer` The {TextBuffer} whose grammar will be set. // * `languageId` The {String} id of the desired language. // // Returns a {Boolean} that indicates whether the language was successfully @@ -398,15 +390,29 @@ class GrammarRegistry { addGrammar (grammar) { if (grammar instanceof TreeSitterGrammar) { this.treeSitterGrammarsById[grammar.id] = grammar + if (grammar.legacyScopeName) { + this.config.addLegacyScopeAlias(grammar.id, grammar.legacyScopeName) + this.textMateScopeNamesByTreeSitterLanguageId.set(grammar.id, grammar.legacyScopeName) + this.treeSitterLanguageIdsByTextMateScopeName.set(grammar.legacyScopeName, grammar.id) + } this.grammarAddedOrUpdated(grammar) - return new Disposable(() => delete this.treeSitterGrammarsById[grammar.id]) + return new Disposable(() => this.removeGrammar(grammar)) } else { return this.textmateRegistry.addGrammar(grammar) } } removeGrammar (grammar) { - return this.textmateRegistry.removeGrammar(grammar) + if (grammar instanceof TreeSitterGrammar) { + delete this.treeSitterGrammarsById[grammar.id] + if (grammar.legacyScopeName) { + this.config.removeLegacyScopeAlias(grammar.id) + this.textMateScopeNamesByTreeSitterLanguageId.delete(grammar.id) + this.treeSitterLanguageIdsByTextMateScopeName.delete(grammar.legacyScopeName) + } + } else { + return this.textmateRegistry.removeGrammar(grammar) + } } removeGrammarForScopeName (scopeName) { @@ -497,11 +503,9 @@ class GrammarRegistry { normalizeLanguageId (languageId) { if (this.config.get('core.useTreeSitterParsers')) { - const row = LANGUAGE_ID_MAP.find(entry => entry[0] === languageId) - return row ? row[1] : languageId + return this.treeSitterLanguageIdsByTextMateScopeName.get(languageId) || languageId } else { - const row = LANGUAGE_ID_MAP.find(entry => entry[1] === languageId) - return row ? row[0] : languageId + return this.textMateScopeNamesByTreeSitterLanguageId.get(languageId) || languageId } } } diff --git a/src/scope-descriptor.coffee b/src/scope-descriptor.coffee index 95539cc69..2085bd6b2 100644 --- a/src/scope-descriptor.coffee +++ b/src/scope-descriptor.coffee @@ -39,11 +39,17 @@ class ScopeDescriptor getScopesArray: -> @scopes getScopeChain: -> - @scopes - .map (scope) -> - scope = ".#{scope}" unless scope[0] is '.' - scope - .join(' ') + # For backward compatibility, prefix TextMate-style scope names with + # leading dots (e.g. 'source.js' -> '.source.js'). + if @scopes[0].includes('.') + result = '' + for scope, i in @scopes + result += ' ' if i > 0 + result += '.' if scope[0] isnt '.' + result += scope + result + else + @scopes.join(' ') toString: -> @getScopeChain() diff --git a/src/tree-sitter-grammar.js b/src/tree-sitter-grammar.js index b36505a0b..d00344fb1 100644 --- a/src/tree-sitter-grammar.js +++ b/src/tree-sitter-grammar.js @@ -8,6 +8,7 @@ class TreeSitterGrammar { this.registry = registry this.id = params.id this.name = params.name + this.legacyScopeName = params.legacyScopeName if (params.contentRegExp) this.contentRegExp = new RegExp(params.contentRegExp) this.folds = params.folds || [] From c844a253e05cc51814f9a2f3359e40b76b1d44d5 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 15 Dec 2017 17:10:20 -0800 Subject: [PATCH 29/39] Implement TreeSitterLanguageMode.scopeDescriptorForPosition --- spec/tree-sitter-language-mode-spec.js | 25 +++++++++++++++++++++++++ src/tree-sitter-language-mode.js | 9 ++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/spec/tree-sitter-language-mode-spec.js b/spec/tree-sitter-language-mode-spec.js index 91070710f..ceb0ec03b 100644 --- a/spec/tree-sitter-language-mode-spec.js +++ b/spec/tree-sitter-language-mode-spec.js @@ -410,6 +410,31 @@ describe('TreeSitterLanguageMode', () => { }) }) + describe('.scopeDescriptorForPosition', () => { + it('returns a scope descriptor representing the given position in the syntax tree', () => { + const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, { + id: 'javascript', + parser: 'tree-sitter-javascript' + }) + + buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar})) + + buffer.setText('foo({bar: baz});') + + editor.screenLineForScreenRow(0) + expect(editor.scopeDescriptorForBufferPosition({row: 0, column: 6}).getScopesArray()).toEqual([ + 'javascript', + 'program', + 'expression_statement', + 'call_expression', + 'arguments', + 'object', + 'pair', + 'property_identifier' + ]) + }) + }) + describe('TextEditor.selectLargerSyntaxNode and .selectSmallerSyntaxNode', () => { it('expands and contract the selection based on the syntax tree', () => { const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, { diff --git a/src/tree-sitter-language-mode.js b/src/tree-sitter-language-mode.js index 5cd725108..310af5fea 100644 --- a/src/tree-sitter-language-mode.js +++ b/src/tree-sitter-language-mode.js @@ -293,7 +293,14 @@ class TreeSitterLanguageMode { } scopeDescriptorForPosition (point) { - return this.rootScopeDescriptor + const result = [] + let node = this.document.rootNode.descendantForPosition(point) + while (node) { + result.push(node.type) + node = node.parent + } + result.push(this.grammar.id) + return new ScopeDescriptor({scopes: result.reverse()}) } hasTokenForSelector (scopeSelector) { From 37cae78bc15f2ab5cfe1c30fc3c4e55152f30443 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Tue, 26 Dec 2017 13:47:34 -0800 Subject: [PATCH 30/39] :arrow_up: tree-sitter and language packages --- package.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/package.json b/package.json index bec57b304..7fb9f62fe 100644 --- a/package.json +++ b/package.json @@ -71,7 +71,7 @@ "sinon": "1.17.4", "temp": "^0.8.3", "text-buffer": "13.9.2", - "tree-sitter": "0.7.5", + "tree-sitter": "^0.8.0", "typescript-simple": "1.0.0", "underscore-plus": "^1.6.6", "winreg": "^1.2.1", @@ -137,18 +137,18 @@ "welcome": "0.36.6", "whitespace": "0.37.5", "wrap-guide": "0.40.3", - "language-c": "0.59.0-2", + "language-c": "0.59.0-3", "language-clojure": "0.22.5", "language-coffee-script": "0.49.3", "language-csharp": "0.14.3", "language-css": "0.42.8", "language-gfm": "0.90.3", "language-git": "0.19.1", - "language-go": "0.45.0-3", + "language-go": "0.45.0-4", "language-html": "0.48.4", "language-hyperlink": "0.16.3", "language-java": "0.27.6", - "language-javascript": "0.128.0-3", + "language-javascript": "0.128.0-4", "language-json": "0.19.1", "language-less": "0.34.1", "language-make": "0.22.3", @@ -157,17 +157,17 @@ "language-perl": "0.38.1", "language-php": "0.43.0", "language-property-list": "0.9.1", - "language-python": "0.46.0-1", + "language-python": "0.46.0-2", "language-ruby": "0.71.4", "language-ruby-on-rails": "0.25.3", "language-sass": "0.61.3", - "language-shellscript": "0.26.0-1", + "language-shellscript": "0.26.0-2", "language-source": "0.9.0", "language-sql": "0.25.9", "language-text": "0.7.3", "language-todo": "0.29.3", "language-toml": "0.18.1", - "language-typescript": "0.3.0-2", + "language-typescript": "0.3.0-3", "language-xml": "0.35.2", "language-yaml": "0.31.1" }, From 662d38135beb8672412ecf24ebf0302e7304494d Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Tue, 26 Dec 2017 14:14:22 -0800 Subject: [PATCH 31/39] Use zero as the minimum value of getGrammarPathScore This way, we can determine if the grammar matches a buffer in any way by checking for a positive score. --- src/grammar-registry.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/grammar-registry.js b/src/grammar-registry.js index b2c4129f7..b316bdbb0 100644 --- a/src/grammar-registry.js +++ b/src/grammar-registry.js @@ -213,7 +213,7 @@ class GrammarRegistry { if (process.platform === 'win32') { filePath = filePath.replace(/\\/g, '/') } const pathComponents = filePath.toLowerCase().split(PATH_SPLIT_REGEX) - let pathScore = -1 + let pathScore = 0 let customFileTypes if (this.config.get('core.customFileTypes')) { From 874e70a3d7a850a05fc5a8455dabe52e9443b9c6 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Tue, 26 Dec 2017 14:14:58 -0800 Subject: [PATCH 32/39] :arrow_up: language-shellscript --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 7fb9f62fe..c3fadd743 100644 --- a/package.json +++ b/package.json @@ -161,7 +161,7 @@ "language-ruby": "0.71.4", "language-ruby-on-rails": "0.25.3", "language-sass": "0.61.3", - "language-shellscript": "0.26.0-2", + "language-shellscript": "0.26.0-3", "language-source": "0.9.0", "language-sql": "0.25.9", "language-text": "0.7.3", From 2da2c1088f49cd2cbecbfc1e033408ea02823114 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 27 Dec 2017 12:28:29 -0800 Subject: [PATCH 33/39] :arrow_up: tree-sitter --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c3fadd743..56ad73666 100644 --- a/package.json +++ b/package.json @@ -71,7 +71,7 @@ "sinon": "1.17.4", "temp": "^0.8.3", "text-buffer": "13.9.2", - "tree-sitter": "^0.8.0", + "tree-sitter": "^0.8.2", "typescript-simple": "1.0.0", "underscore-plus": "^1.6.6", "winreg": "^1.2.1", From a8e457df61168d8e85d3829a33088e1df6f49036 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 27 Dec 2017 12:37:50 -0800 Subject: [PATCH 34/39] Tweak syntax selection key bindings --- keymaps/darwin.cson | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/keymaps/darwin.cson b/keymaps/darwin.cson index d5cc7b7da..6d576f102 100644 --- a/keymaps/darwin.cson +++ b/keymaps/darwin.cson @@ -133,6 +133,8 @@ 'cmd-ctrl-left': 'editor:move-selection-left' 'cmd-ctrl-right': 'editor:move-selection-right' 'cmd-shift-V': 'editor:paste-without-reformatting' + 'alt-up': 'editor:select-larger-syntax-node' + 'alt-down': 'editor:select-smaller-syntax-node' # Emacs 'alt-f': 'editor:move-to-end-of-word' @@ -161,8 +163,6 @@ 'ctrl-alt-shift-right': 'editor:select-to-next-subword-boundary' 'ctrl-alt-backspace': 'editor:delete-to-beginning-of-subword' 'ctrl-alt-delete': 'editor:delete-to-end-of-subword' - 'ctrl-alt-up': 'editor:select-larger-syntax-node' - 'ctrl-alt-down': 'editor:select-smaller-syntax-node' 'atom-workspace atom-text-editor:not([mini])': # Atom specific From 7e0f4f377ef588ff7167bad5a83a0767bc48ce96 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Sat, 30 Dec 2017 21:55:13 -0800 Subject: [PATCH 35/39] :arrow_up: tree-sitter --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e047c40c0..799fa0a1d 100644 --- a/package.json +++ b/package.json @@ -71,7 +71,7 @@ "sinon": "1.17.4", "temp": "^0.8.3", "text-buffer": "13.9.2", - "tree-sitter": "^0.8.2", + "tree-sitter": "^0.8.4", "typescript-simple": "1.0.0", "underscore-plus": "^1.6.6", "winreg": "^1.2.1", From 629cb206ec77bed393b58e51d34605aece9bf6c5 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 3 Jan 2018 09:34:12 -0800 Subject: [PATCH 36/39] Fix handling of empty tokens in TreeSitterHighlightIterator --- spec/tree-sitter-language-mode-spec.js | 45 ++++++++++++++++++++++++++ src/tree-sitter-language-mode.js | 6 +++- 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/spec/tree-sitter-language-mode-spec.js b/spec/tree-sitter-language-mode-spec.js index ceb0ec03b..ec38c1a06 100644 --- a/spec/tree-sitter-language-mode-spec.js +++ b/spec/tree-sitter-language-mode-spec.js @@ -2,6 +2,7 @@ const {it, fit, ffit, fffit, beforeEach, afterEach} = require('./async-spec-help const dedent = require('dedent') const TextBuffer = require('text-buffer') +const {Point} = TextBuffer const TextEditor = require('../src/text-editor') const TreeSitterGrammar = require('../src/tree-sitter-grammar') const TreeSitterLanguageMode = require('../src/tree-sitter-language-mode') @@ -93,6 +94,50 @@ describe('TreeSitterLanguageMode', () => { ] ]) }) + + it('correctly skips over tokens with zero size', () => { + const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, { + parser: 'tree-sitter-c', + scopes: { + 'primitive_type': 'type', + 'identifier': 'variable', + } + }) + + const languageMode = new TreeSitterLanguageMode({buffer, grammar}) + buffer.setLanguageMode(languageMode) + buffer.setText('int main() {\n int a\n int b;\n}'); + + editor.screenLineForScreenRow(0) + expect( + languageMode.document.rootNode.descendantForPosition(Point(1, 2), Point(1, 6)).toString() + ).toBe('(declaration (primitive_type) (identifier) (MISSING))') + + expectTokensToEqual(editor, [ + [ + {text: 'int', scopes: ['type']}, + {text: ' ', scopes: []}, + {text: 'main', scopes: ['variable']}, + {text: '() {', scopes: []} + ], + [ + {text: ' ', scopes: ['whitespace']}, + {text: 'int', scopes: ['type']}, + {text: ' ', scopes: []}, + {text: 'a', scopes: ['variable']} + ], + [ + {text: ' ', scopes: ['whitespace']}, + {text: 'int', scopes: ['type']}, + {text: ' ', scopes: []}, + {text: 'b', scopes: ['variable']}, + {text: ';', scopes: []} + ], + [ + {text: '}', scopes: []} + ] + ]) + }) }) describe('folding', () => { diff --git a/src/tree-sitter-language-mode.js b/src/tree-sitter-language-mode.js index 310af5fea..8cba4e25f 100644 --- a/src/tree-sitter-language-mode.js +++ b/src/tree-sitter-language-mode.js @@ -422,7 +422,7 @@ class TreeSitterHighlightIterator { if (!this.currentNode) break } } - } else { + } else if (this.currentNode.startIndex < this.currentNode.endIndex) { this.currentNode = this.currentNode.nextSibling if (this.currentNode) { this.currentChildIndex++ @@ -431,6 +431,10 @@ class TreeSitterHighlightIterator { this.pushOpenTag() this.descendLeft() } + } else { + this.pushCloseTag() + this.currentNode = this.currentNode.parent + this.currentChildIndex = last(this.containingNodeChildIndices) } } while (this.closeTags.length === 0 && this.openTags.length === 0 && this.currentNode) From 7f923fc05fdc1001007dbf92eee4de97bd0bef0f Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 4 Jan 2018 12:13:23 -0800 Subject: [PATCH 37/39] Fix section comments --- src/tree-sitter-language-mode.js | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/tree-sitter-language-mode.js b/src/tree-sitter-language-mode.js index 8cba4e25f..313c3574d 100644 --- a/src/tree-sitter-language-mode.js +++ b/src/tree-sitter-language-mode.js @@ -41,8 +41,8 @@ class TreeSitterLanguageMode { } /* - * Section - Highlighting - */ + Section - Highlighting + */ buildHighlightIterator () { const invalidatedRanges = this.document.parse() @@ -67,8 +67,8 @@ class TreeSitterLanguageMode { } /* - * Section - Commenting - */ + Section - Commenting + */ commentStringsForPosition () { return this.grammar.commentStrings @@ -79,8 +79,8 @@ class TreeSitterLanguageMode { } /* - * Section - Indentation - */ + Section - Indentation + */ suggestedIndentForLineAtBufferRow (row, line, tabLength) { return this.suggestedIndentForBufferRow(row, tabLength) @@ -119,8 +119,8 @@ class TreeSitterLanguageMode { } /* - * Section - Folding - */ + Section - Folding + */ isFoldableAtRow (row) { if (this.isFoldableCache[row] != null) return this.isFoldableCache[row] @@ -263,8 +263,8 @@ class TreeSitterLanguageMode { } /* - * Syntax Tree APIs - */ + Syntax Tree APIs + */ getRangeForSyntaxNodeContainingRange (range) { const startIndex = this.buffer.characterIndexForPosition(range.start) @@ -277,8 +277,8 @@ class TreeSitterLanguageMode { } /* - * Section - Backward compatibility shims - */ + Section - Backward compatibility shims + */ tokenizedLineForRow (row) { return new TokenizedLine({ From 3f11fa57ee7022b5383070427a91f82238cfa1ed Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 5 Jan 2018 20:26:41 -0800 Subject: [PATCH 38/39] Make tree-sitter indent methods delegate to textmate ones for now --- src/text-mate-language-mode.js | 27 +++++++++++-------- src/tree-sitter-language-mode.js | 45 +++++++++++++++++++++----------- 2 files changed, 46 insertions(+), 26 deletions(-) diff --git a/src/text-mate-language-mode.js b/src/text-mate-language-mode.js index 123e39f58..1a7cb6d2e 100644 --- a/src/text-mate-language-mode.js +++ b/src/text-mate-language-mode.js @@ -74,10 +74,15 @@ class TextMateLanguageMode { // // Returns a {Number}. suggestedIndentForBufferRow (bufferRow, tabLength, options) { - return this._suggestedIndentForTokenizedLineAtBufferRow( + const line = this.buffer.lineForRow(bufferRow) + const tokenizedLine = this.tokenizedLineForRow(bufferRow) + const iterator = tokenizedLine.getTokenIterator() + iterator.next() + const scopeDescriptor = new ScopeDescriptor({scopes: iterator.getScopes()}) + return this._suggestedIndentForLineWithScopeAtBufferRow( bufferRow, - this.buffer.lineForRow(bufferRow), - this.tokenizedLineForRow(bufferRow), + line, + scopeDescriptor, tabLength, options ) @@ -90,10 +95,14 @@ class TextMateLanguageMode { // // Returns a {Number}. suggestedIndentForLineAtBufferRow (bufferRow, line, tabLength) { - return this._suggestedIndentForTokenizedLineAtBufferRow( + const tokenizedLine = this.buildTokenizedLineForRowWithText(bufferRow, line) + const iterator = tokenizedLine.getTokenIterator() + iterator.next() + const scopeDescriptor = new ScopeDescriptor({scopes: iterator.getScopes()}) + return this._suggestedIndentForLineWithScopeAtBufferRow( bufferRow, line, - this.buildTokenizedLineForRowWithText(bufferRow, line), + scopeDescriptor, tabLength ) } @@ -111,7 +120,7 @@ class TextMateLanguageMode { const currentIndentLevel = this.indentLevelForLine(line, tabLength) if (currentIndentLevel === 0) return - const scopeDescriptor = this.scopeDescriptorForPosition([bufferRow, 0]) + const scopeDescriptor = this.scopeDescriptorForPosition(new Point(bufferRow, 0)) const decreaseIndentRegex = this.decreaseIndentRegexForScopeDescriptor(scopeDescriptor) if (!decreaseIndentRegex) return @@ -138,11 +147,7 @@ class TextMateLanguageMode { return desiredIndentLevel } - _suggestedIndentForTokenizedLineAtBufferRow (bufferRow, line, tokenizedLine, tabLength, options) { - const iterator = tokenizedLine.getTokenIterator() - iterator.next() - const scopeDescriptor = new ScopeDescriptor({scopes: iterator.getScopes()}) - + _suggestedIndentForLineWithScopeAtBufferRow (bufferRow, line, scopeDescriptor, tabLength, options) { const increaseIndentRegex = this.increaseIndentRegexForScopeDescriptor(scopeDescriptor) const decreaseIndentRegex = this.decreaseIndentRegexForScopeDescriptor(scopeDescriptor) const decreaseNextIndentRegex = this.decreaseNextIndentRegexForScopeDescriptor(scopeDescriptor) diff --git a/src/tree-sitter-language-mode.js b/src/tree-sitter-language-mode.js index 313c3574d..2ab023b86 100644 --- a/src/tree-sitter-language-mode.js +++ b/src/tree-sitter-language-mode.js @@ -2,6 +2,7 @@ const {Document} = require('tree-sitter') const {Point, Range, Emitter} = require('atom') const ScopeDescriptor = require('./scope-descriptor') const TokenizedLine = require('./tokenized-line') +const TextMateLanguageMode = require('./text-mate-language-mode') let nextId = 0 @@ -19,6 +20,10 @@ class TreeSitterLanguageMode { this.rootScopeDescriptor = new ScopeDescriptor({scopes: [this.grammar.id]}) this.emitter = new Emitter() this.isFoldableCache = [] + + // TODO: Remove this once TreeSitterLanguageMode implements its own auto-indentation system. This + // is temporarily needed in order to delegate to the TextMateLanguageMode's auto-indent system. + this.regexesByPattern = {} } getLanguageId () { @@ -83,24 +88,22 @@ class TreeSitterLanguageMode { */ suggestedIndentForLineAtBufferRow (row, line, tabLength) { - return this.suggestedIndentForBufferRow(row, tabLength) + return this._suggestedIndentForLineWithScopeAtBufferRow( + row, + line, + this.rootScopeDescriptor, + tabLength + ) } suggestedIndentForBufferRow (row, tabLength, options) { - let precedingRow - if (!options || options.skipBlankLines !== false) { - precedingRow = this.buffer.previousNonBlankRow(row) - if (precedingRow == null) return 0 - } else { - precedingRow = row - 1 - if (precedingRow < 0) return 0 - } - - return this.indentLevelForLine(this.buffer.lineForRow(precedingRow), tabLength) - } - - suggestedIndentForEditedBufferRow (row) { - return null + return this._suggestedIndentForLineWithScopeAtBufferRow( + row, + this.buffer.lineForRow(row), + this.rootScopeDescriptor, + tabLength, + options + ) } indentLevelForLine (line, tabLength = tabLength) { @@ -508,3 +511,15 @@ class TreeSitterTextBufferInput { function last (array) { return array[array.length - 1] } + +// TODO: Remove this once TreeSitterLanguageMode implements its own auto-indent system. +[ + '_suggestedIndentForLineWithScopeAtBufferRow', + 'suggestedIndentForEditedBufferRow', + 'increaseIndentRegexForScopeDescriptor', + 'decreaseIndentRegexForScopeDescriptor', + 'decreaseNextIndentRegexForScopeDescriptor', + 'regexForPattern' +].forEach(methodName => { + module.exports.prototype[methodName] = TextMateLanguageMode.prototype[methodName] +}) From 84c9524403d115c7a0a3505880655c3befe52103 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 5 Jan 2018 20:26:41 -0800 Subject: [PATCH 39/39] Omit anonymous token types in tree-sitter scope descriptors for now --- src/tree-sitter-language-mode.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/tree-sitter-language-mode.js b/src/tree-sitter-language-mode.js index 2ab023b86..41c87ba00 100644 --- a/src/tree-sitter-language-mode.js +++ b/src/tree-sitter-language-mode.js @@ -298,6 +298,13 @@ class TreeSitterLanguageMode { scopeDescriptorForPosition (point) { const result = [] let node = this.document.rootNode.descendantForPosition(point) + + // Don't include anonymous token types like '(' because they prevent scope chains + // from being parsed as CSS selectors by the `slick` parser. Other css selector + // parsers like `postcss-selector-parser` do allow arbitrary quoted strings in + // selectors. + if (!node.isNamed) node = node.parent + while (node) { result.push(node.type) node = node.parent