diff --git a/package.json b/package.json
index 2c8efb2d1..0ed8ce98e 100644
--- a/package.json
+++ b/package.json
@@ -157,9 +157,9 @@
"symbols-view": "https://www.atom.io/api/packages/symbols-view/versions/0.118.2/tarball",
"tabs": "https://www.atom.io/api/packages/tabs/versions/0.109.2/tarball",
"temp": "^0.8.3",
- "text-buffer": "13.14.4",
+ "text-buffer": "13.14.5",
"timecop": "https://www.atom.io/api/packages/timecop/versions/0.36.2/tarball",
- "tree-sitter": "0.12.12",
+ "tree-sitter": "0.12.20",
"tree-view": "https://www.atom.io/api/packages/tree-view/versions/0.222.0/tarball",
"typescript-simple": "1.0.0",
"underscore-plus": "^1.6.8",
@@ -203,7 +203,7 @@
"exception-reporting": "0.43.1",
"find-and-replace": "0.215.11",
"fuzzy-finder": "1.8.2",
- "github": "0.17.3",
+ "github": "0.18.2",
"git-diff": "1.3.9",
"go-to-line": "0.33.0",
"grammar-selector": "0.50.1",
@@ -238,10 +238,10 @@
"language-gfm": "0.90.5",
"language-git": "0.19.1",
"language-go": "0.45.4",
- "language-html": "0.50.1",
+ "language-html": "0.50.3",
"language-hyperlink": "0.16.3",
"language-java": "0.30.0",
- "language-javascript": "0.128.8",
+ "language-javascript": "0.128.11",
"language-json": "0.19.2",
"language-less": "0.34.2",
"language-make": "0.22.3",
diff --git a/spec/grammar-registry-spec.js b/spec/grammar-registry-spec.js
index abb3b189a..bcd57f3a2 100644
--- a/spec/grammar-registry-spec.js
+++ b/spec/grammar-registry-spec.js
@@ -525,6 +525,34 @@ describe('GrammarRegistry', () => {
})
})
+ describe('.addInjectionPoint(languageId, {type, language, content})', () => {
+ const injectionPoint = {
+ type: 'some_node_type',
+ language() { return 'some_language_name' },
+ content(node) { return node }
+ }
+
+ beforeEach(() => {
+ atom.config.set('core.useTreeSitterParsers', true)
+ })
+
+ it('adds an injection point to the grammar with the given id', async () => {
+ await atom.packages.activatePackage('language-javascript')
+ atom.grammars.addInjectionPoint('javascript', injectionPoint)
+ const grammar = atom.grammars.grammarForId('javascript')
+ expect(grammar.injectionPoints).toContain(injectionPoint)
+ })
+
+ describe('when called before a grammar with the given id is loaded', () => {
+ it('adds the injection point once the grammar is loaded', async () => {
+ atom.grammars.addInjectionPoint('javascript', injectionPoint)
+ await atom.packages.activatePackage('language-javascript')
+ const grammar = atom.grammars.grammarForId('javascript')
+ expect(grammar.injectionPoints).toContain(injectionPoint)
+ })
+ })
+ })
+
describe('serialization', () => {
it('persists editors\' grammar overrides', async () => {
const buffer1 = new TextBuffer()
diff --git a/spec/tree-sitter-language-mode-spec.js b/spec/tree-sitter-language-mode-spec.js
index 014d8122e..c23849d30 100644
--- a/spec/tree-sitter-language-mode-spec.js
+++ b/spec/tree-sitter-language-mode-spec.js
@@ -11,6 +11,7 @@ const cGrammarPath = require.resolve('language-c/grammars/tree-sitter-c.cson')
const pythonGrammarPath = require.resolve('language-python/grammars/tree-sitter-python.cson')
const jsGrammarPath = require.resolve('language-javascript/grammars/tree-sitter-javascript.cson')
const htmlGrammarPath = require.resolve('language-html/grammars/tree-sitter-html.cson')
+const ejsGrammarPath = require.resolve('language-html/grammars/tree-sitter-ejs.cson')
describe('TreeSitterLanguageMode', () => {
let editor, buffer
@@ -18,6 +19,7 @@ describe('TreeSitterLanguageMode', () => {
beforeEach(async () => {
editor = await atom.workspace.open('')
buffer = editor.getBuffer()
+ editor.displayLayer.reset({foldCharacter: '…'})
})
describe('highlighting', () => {
@@ -32,10 +34,11 @@ describe('TreeSitterLanguageMode', () => {
}
})
+ buffer.setText('aa.bbb = cc(d.eee());')
+
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
- buffer.setText('aa.bbb = cc(d.eee());')
- await languageMode.reparsePromise
+ await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [[
{text: 'aa.', scopes: ['source']},
@@ -61,10 +64,11 @@ describe('TreeSitterLanguageMode', () => {
}
})
+ buffer.setText('a = bb.ccc();')
+
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
- buffer.setText('a = bb.ccc();')
- await languageMode.reparsePromise
+ await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [[
{text: 'a', scopes: ['source', 'variable']},
@@ -87,17 +91,18 @@ describe('TreeSitterLanguageMode', () => {
}
})
+ buffer.setText('a\n .b();')
+
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
- buffer.setText('a\n .b();')
- await languageMode.reparsePromise
+ await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [
[
{text: 'a', scopes: ['variable']},
],
[
- {text: ' ', scopes: ['whitespace']},
+ {text: ' ', scopes: ['leading-whitespace']},
{text: '.', scopes: []},
{text: 'b', scopes: ['function']},
{text: '();', scopes: []}
@@ -114,12 +119,12 @@ describe('TreeSitterLanguageMode', () => {
}
})
+ buffer.setText('int main() {\n int a\n int b;\n}');
+
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
- buffer.setText('int main() {\n int a\n int b;\n}');
- await languageMode.reparsePromise
+ await nextHighlightingUpdate(languageMode)
- editor.screenLineForScreenRow(0)
expect(
languageMode.tree.rootNode.descendantForPosition(Point(1, 2), Point(1, 6)).toString()
).toBe('(declaration (primitive_type) (identifier) (MISSING))')
@@ -132,13 +137,13 @@ describe('TreeSitterLanguageMode', () => {
{text: '() {', scopes: []}
],
[
- {text: ' ', scopes: ['whitespace']},
+ {text: ' ', scopes: ['leading-whitespace']},
{text: 'int', scopes: ['type']},
{text: ' ', scopes: []},
{text: 'a', scopes: ['variable']}
],
[
- {text: ' ', scopes: ['whitespace']},
+ {text: ' ', scopes: ['leading-whitespace']},
{text: 'int', scopes: ['type']},
{text: ' ', scopes: []},
{text: 'b', scopes: ['variable']},
@@ -159,10 +164,11 @@ describe('TreeSitterLanguageMode', () => {
}
})
+ buffer.setText('a(\nb,\nc\n')
+
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
- buffer.setText('a(\nb,\nc\n')
- await languageMode.reparsePromise
+ await nextHighlightingUpdate(languageMode)
// missing closing paren
expectTokensToEqual(editor, [
@@ -173,7 +179,7 @@ describe('TreeSitterLanguageMode', () => {
])
buffer.append(')')
- await languageMode.reparsePromise
+ await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [
[
{text: 'a', scopes: ['function']},
@@ -195,15 +201,16 @@ describe('TreeSitterLanguageMode', () => {
}
})
- const languageMode = new TreeSitterLanguageMode({buffer, grammar})
- buffer.setLanguageMode(languageMode)
buffer.setText([
'// abc',
'',
'a("b").c'
].join('\r\n'))
- await languageMode.reparsePromise
+ const languageMode = new TreeSitterLanguageMode({buffer, grammar})
+ buffer.setLanguageMode(languageMode)
+
+ await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [
[{text: '// abc', scopes: ['comment']}],
[{text: '', scopes: []}],
@@ -216,12 +223,12 @@ describe('TreeSitterLanguageMode', () => {
])
buffer.insert([2, 0], ' ')
- await languageMode.reparsePromise
+ await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [
[{text: '// abc', scopes: ['comment']}],
[{text: '', scopes: []}],
[
- {text: ' ', scopes: ['whitespace']},
+ {text: ' ', scopes: ['leading-whitespace']},
{text: 'a(', scopes: []},
{text: '"b"', scopes: ['string']},
{text: ').', scopes: []},
@@ -230,6 +237,82 @@ describe('TreeSitterLanguageMode', () => {
])
})
+ it('handles multi-line nodes with children on different lines (regression)', async () => {
+ const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
+ parser: 'tree-sitter-javascript',
+ scopes: {
+ 'template_string': 'string',
+ '"${"': 'interpolation',
+ '"}"': 'interpolation'
+ }
+ });
+
+ buffer.setText('`\na${1}\nb${2}\n`;')
+
+ const languageMode = new TreeSitterLanguageMode({buffer, grammar})
+ buffer.setLanguageMode(languageMode)
+ await nextHighlightingUpdate(languageMode)
+
+ expectTokensToEqual(editor, [
+ [
+ {text: '`', scopes: ['string']}
+ ], [
+ {text: 'a', scopes: ['string']},
+ {text: '${', scopes: ['string', 'interpolation']},
+ {text: '1', scopes: ['string']},
+ {text: '}', scopes: ['string', 'interpolation']}
+ ], [
+ {text: 'b', scopes: ['string']},
+ {text: '${', scopes: ['string', 'interpolation']},
+ {text: '2', scopes: ['string']},
+ {text: '}', scopes: ['string', 'interpolation']}
+ ],
+ [
+ {text: '`', scopes: ['string']},
+ {text: ';', scopes: []}
+ ]
+ ])
+ })
+
+ it('handles folds inside of highlighted tokens', async () => {
+ const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
+ parser: 'tree-sitter-javascript',
+ scopes: {
+ 'comment': 'comment',
+ 'call_expression > identifier': 'function',
+ }
+ })
+
+ buffer.setText(dedent `
+ /*
+ * Hello
+ */
+
+ hello();
+ `)
+
+ const languageMode = new TreeSitterLanguageMode({buffer, grammar})
+ buffer.setLanguageMode(languageMode)
+ await nextHighlightingUpdate(languageMode)
+
+ editor.foldBufferRange([[0, 2], [2, 0]])
+
+ expectTokensToEqual(editor, [
+ [
+ {text: '/*', scopes: ['comment']},
+ {text: '…', scopes: ['fold-marker']},
+ {text: ' */', scopes: ['comment']}
+ ],
+ [
+ {text: '', scopes: []}
+ ],
+ [
+ {text: 'hello', scopes: ['function']},
+ {text: '();', scopes: []},
+ ]
+ ])
+ })
+
describe('when the buffer changes during a parse', () => {
it('immediately parses again when the current parse completes', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
@@ -240,11 +323,14 @@ describe('TreeSitterLanguageMode', () => {
'new_expression > call_expression > identifier': 'constructor'
}
})
- const languageMode = new TreeSitterLanguageMode({buffer, grammar})
- buffer.setLanguageMode(languageMode)
buffer.setText('abc;');
- await languageMode.reparsePromise
+
+ const languageMode = new TreeSitterLanguageMode({buffer, grammar})
+ buffer.setLanguageMode(languageMode)
+ await nextHighlightingUpdate(languageMode)
+ await new Promise(process.nextTick)
+
expectTokensToEqual(editor, [
[
{text: 'abc', scopes: ['variable']},
@@ -269,8 +355,7 @@ describe('TreeSitterLanguageMode', () => {
],
])
- await languageMode.reparsePromise
- expect(languageMode.reparsePromise).not.toBeNull()
+ await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [
[
{text: 'new ', scopes: []},
@@ -279,8 +364,7 @@ describe('TreeSitterLanguageMode', () => {
],
])
- await languageMode.reparsePromise
- expect(languageMode.reparsePromise).toBeNull()
+ await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [
[
{text: 'new ', scopes: []},
@@ -290,13 +374,265 @@ describe('TreeSitterLanguageMode', () => {
])
})
})
+
+ describe('injectionPoints and injectionPatterns', () => {
+ let jsGrammar, htmlGrammar
+
+ beforeEach(() => {
+ jsGrammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
+ id: 'javascript',
+ parser: 'tree-sitter-javascript',
+ scopes: {
+ 'property_identifier': 'property',
+ 'call_expression > identifier': 'function',
+ 'template_string': 'string',
+ 'template_substitution > "${"': 'interpolation',
+ 'template_substitution > "}"': 'interpolation'
+ },
+ injectionRegExp: 'javascript',
+ injectionPoints: [HTML_TEMPLATE_LITERAL_INJECTION_POINT]
+ })
+
+ htmlGrammar = new TreeSitterGrammar(atom.grammars, htmlGrammarPath, {
+ id: 'html',
+ parser: 'tree-sitter-html',
+ scopes: {
+ fragment: 'html',
+ tag_name: 'tag',
+ attribute_name: 'attr'
+ },
+ injectionRegExp: 'html',
+ injectionPoints: [SCRIPT_TAG_INJECTION_POINT]
+ })
+ })
+
+ it('highlights code inside of injection points', async () => {
+ atom.grammars.addGrammar(jsGrammar)
+ atom.grammars.addGrammar(htmlGrammar)
+ buffer.setText('node.innerHTML = html `\na ${b}
\n`;')
+
+ const languageMode = new TreeSitterLanguageMode({buffer, grammar: jsGrammar, grammars: atom.grammars})
+ buffer.setLanguageMode(languageMode)
+ await nextHighlightingUpdate(languageMode)
+ await nextHighlightingUpdate(languageMode)
+
+ expectTokensToEqual(editor, [
+ [
+ {text: 'node.', scopes: []},
+ {text: 'innerHTML', scopes: ['property']},
+ {text: ' = ', scopes: []},
+ {text: 'html', scopes: ['function']},
+ {text: ' ', scopes: []},
+ {text: '`', scopes: ['string']},
+ {text: '', scopes: ['string', 'html']}
+ ], [
+ {text: 'a ', scopes: ['string', 'html']},
+ {text: '${', scopes: ['string', 'html', 'interpolation']},
+ {text: 'b', scopes: ['string', 'html']},
+ {text: '}', scopes: ['string', 'html', 'interpolation']},
+ {text: '<', scopes: ['string', 'html']},
+ {text: 'img', scopes: ['string', 'html', 'tag']},
+ {text: ' ', scopes: ['string', 'html']},
+ {text: 'src', scopes: ['string', 'html', 'attr']},
+ {text: '="d">', scopes: ['string', 'html']}
+ ], [
+ {text: '`', scopes: ['string']},
+ {text: ';', scopes: []},
+ ],
+ ])
+
+ const range = buffer.findSync('html')
+ buffer.setTextInRange(range, 'xml')
+ await nextHighlightingUpdate(languageMode)
+ await nextHighlightingUpdate(languageMode)
+
+ expectTokensToEqual(editor, [
+ [
+ {text: 'node.', scopes: []},
+ {text: 'innerHTML', scopes: ['property']},
+ {text: ' = ', scopes: []},
+ {text: 'xml', scopes: ['function']},
+ {text: ' ', scopes: []},
+ {text: '`', scopes: ['string']}
+ ], [
+ {text: 'a ', scopes: ['string']},
+ {text: '${', scopes: ['string', 'interpolation']},
+ {text: 'b', scopes: ['string']},
+ {text: '}', scopes: ['string', 'interpolation']},
+ {text: '', scopes: ['string']},
+ ], [
+ {text: '`', scopes: ['string']},
+ {text: ';', scopes: []},
+ ],
+ ])
+ })
+
+ it('highlights the content after injections', async () => {
+ atom.grammars.addGrammar(jsGrammar)
+ atom.grammars.addGrammar(htmlGrammar)
+ buffer.setText('\n\n
')
+
+ const languageMode = new TreeSitterLanguageMode({buffer, grammar: htmlGrammar, grammars: atom.grammars})
+ buffer.setLanguageMode(languageMode)
+ await nextHighlightingUpdate(languageMode)
+ await nextHighlightingUpdate(languageMode)
+
+ expectTokensToEqual(editor, [
+ [
+ {text: '<', scopes: ['html']},
+ {text: 'script', scopes: ['html', 'tag']},
+ {text: '>', scopes: ['html']},
+ ],
+ [
+ {text: 'hello', scopes: ['html', 'function']},
+ {text: '();', scopes: ['html']},
+ ],
+ [
+ {text: '', scopes: ['html']},
+ ],
+ [
+ {text: '<', scopes: ['html']},
+ {text: 'div', scopes: ['html', 'tag']},
+ {text: '>', scopes: ['html']},
+ ],
+ [
+ {text: '', scopes: ['html']},
+ ]
+ ])
+ })
+
+ it('updates buffers highlighting when a grammar with injectionRegExp is added', async () => {
+ atom.grammars.addGrammar(jsGrammar)
+
+ buffer.setText('node.innerHTML = html `\na ${b}\n`;')
+ const languageMode = new TreeSitterLanguageMode({buffer, grammar: jsGrammar, grammars: atom.grammars})
+ buffer.setLanguageMode(languageMode)
+
+ await nextHighlightingUpdate(languageMode)
+ expectTokensToEqual(editor, [
+ [
+ {text: 'node.', scopes: []},
+ {text: 'innerHTML', scopes: ['property']},
+ {text: ' = ', scopes: []},
+ {text: 'html', scopes: ['function']},
+ {text: ' ', scopes: []},
+ {text: '`', scopes: ['string']}
+ ], [
+ {text: 'a ', scopes: ['string']},
+ {text: '${', scopes: ['string', 'interpolation']},
+ {text: 'b', scopes: ['string']},
+ {text: '}', scopes: ['string', 'interpolation']},
+ {text: '', scopes: ['string']},
+ ], [
+ {text: '`', scopes: ['string']},
+ {text: ';', scopes: []},
+ ],
+ ])
+
+ atom.grammars.addGrammar(htmlGrammar)
+ await nextHighlightingUpdate(languageMode)
+ expectTokensToEqual(editor, [
+ [
+ {text: 'node.', scopes: []},
+ {text: 'innerHTML', scopes: ['property']},
+ {text: ' = ', scopes: []},
+ {text: 'html', scopes: ['function']},
+ {text: ' ', scopes: []},
+ {text: '`', scopes: ['string']},
+ {text: '', scopes: ['string', 'html']}
+ ], [
+ {text: 'a ', scopes: ['string', 'html']},
+ {text: '${', scopes: ['string', 'html', 'interpolation']},
+ {text: 'b', scopes: ['string', 'html']},
+ {text: '}', scopes: ['string', 'html', 'interpolation']},
+ {text: '<', scopes: ['string', 'html']},
+ {text: 'img', scopes: ['string', 'html', 'tag']},
+ {text: ' ', scopes: ['string', 'html']},
+ {text: 'src', scopes: ['string', 'html', 'attr']},
+ {text: '="d">', scopes: ['string', 'html']}
+ ], [
+ {text: '`', scopes: ['string']},
+ {text: ';', scopes: []},
+ ],
+ ])
+ })
+
+ it('handles injections that intersect', async () => {
+ const ejsGrammar = new TreeSitterGrammar(atom.grammars, ejsGrammarPath, {
+ id: 'ejs',
+ parser: 'tree-sitter-embedded-template',
+ scopes: {
+ '"<%="': 'directive',
+ '"%>"': 'directive',
+ },
+ injectionPoints: [
+ {
+ type: 'template',
+ language (node) { return 'javascript' },
+ content (node) { return node.descendantsOfType('code') }
+ },
+ {
+ type: 'template',
+ language (node) { return 'html' },
+ content (node) { return node.descendantsOfType('content') }
+ }
+ ]
+ })
+
+ atom.grammars.addGrammar(jsGrammar)
+ atom.grammars.addGrammar(htmlGrammar)
+
+ buffer.setText('\n\n')
+ const languageMode = new TreeSitterLanguageMode({buffer, grammar: ejsGrammar, grammars: atom.grammars})
+ buffer.setLanguageMode(languageMode)
+
+ // 4 parses: EJS, HTML, template JS, script tag JS
+ await nextHighlightingUpdate(languageMode)
+ await nextHighlightingUpdate(languageMode)
+ await nextHighlightingUpdate(languageMode)
+ await nextHighlightingUpdate(languageMode)
+
+ expectTokensToEqual(editor, [
+ [
+ {text: '<', scopes: ['html']},
+ {text: 'body', scopes: ['html', 'tag']},
+ {text: '>', scopes: ['html']}
+ ],
+ [
+ {text: '<', scopes: ['html']},
+ {text: 'script', scopes: ['html', 'tag']},
+ {text: '>', scopes: ['html']}
+ ],
+ [
+ {text: 'b', scopes: ['html', 'function']},
+ {text: '(', scopes: ['html']},
+ {text: '<%=', scopes: ['html', 'directive']},
+ {text: ' c.', scopes: ['html']},
+ {text: 'd', scopes: ['html', 'property']},
+ {text: ' ', scopes: ['html']},
+ {text: '%>', scopes: ['html', 'directive']},
+ {text: ')', scopes: ['html']},
+ ],
+ [
+ {text: '', scopes: ['html']}
+ ],
+ [
+ {text: '', scopes: ['html']}
+ ],
+ ])
+ })
+ })
})
describe('folding', () => {
- beforeEach(() => {
- editor.displayLayer.reset({foldCharacter: '…'})
- })
-
it('can fold nodes that start and end with specified tokens', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
@@ -312,8 +648,6 @@ describe('TreeSitterLanguageMode', () => {
]
})
- const languageMode = new TreeSitterLanguageMode({buffer, grammar})
- buffer.setLanguageMode(languageMode)
buffer.setText(dedent `
module.exports =
class A {
@@ -324,9 +658,10 @@ describe('TreeSitterLanguageMode', () => {
}
}
`)
- await languageMode.reparsePromise
- editor.screenLineForScreenRow(0)
+ const languageMode = new TreeSitterLanguageMode({buffer, grammar})
+ buffer.setLanguageMode(languageMode)
+ await nextHighlightingUpdate(languageMode)
expect(editor.isFoldableAtBufferRow(0)).toBe(false)
expect(editor.isFoldableAtBufferRow(1)).toBe(true)
@@ -375,8 +710,6 @@ describe('TreeSitterLanguageMode', () => {
]
})
- const languageMode = new TreeSitterLanguageMode({buffer, grammar})
- buffer.setLanguageMode(languageMode)
buffer.setText(dedent `
const element1 = {
world
`)
- await languageMode.reparsePromise
- editor.screenLineForScreenRow(0)
+ const languageMode = new TreeSitterLanguageMode({buffer, grammar})
+ buffer.setLanguageMode(languageMode)
+ await nextHighlightingUpdate(languageMode)
expect(editor.isFoldableAtBufferRow(0)).toBe(true)
expect(editor.isFoldableAtBufferRow(1)).toBe(false)
@@ -427,8 +761,6 @@ describe('TreeSitterLanguageMode', () => {
]
})
- const languageMode = new TreeSitterLanguageMode({buffer, grammar})
- buffer.setLanguageMode(languageMode)
buffer.setText(dedent `
/**
* Important
@@ -437,9 +769,10 @@ describe('TreeSitterLanguageMode', () => {
Also important
*/
`)
- await languageMode.reparsePromise
- editor.screenLineForScreenRow(0)
+ const languageMode = new TreeSitterLanguageMode({buffer, grammar})
+ buffer.setLanguageMode(languageMode)
+ await nextHighlightingUpdate(languageMode)
expect(editor.isFoldableAtBufferRow(0)).toBe(true)
expect(editor.isFoldableAtBufferRow(1)).toBe(false)
@@ -488,8 +821,6 @@ describe('TreeSitterLanguageMode', () => {
]
})
- const languageMode = new TreeSitterLanguageMode({buffer, grammar})
- buffer.setLanguageMode(languageMode)
buffer.setText(dedent `
#ifndef FOO_H_
#define FOO_H_
@@ -513,9 +844,10 @@ describe('TreeSitterLanguageMode', () => {
#endif
`)
- await languageMode.reparsePromise
- editor.screenLineForScreenRow(0)
+ const languageMode = new TreeSitterLanguageMode({buffer, grammar})
+ buffer.setLanguageMode(languageMode)
+ await nextHighlightingUpdate(languageMode)
editor.foldBufferRow(3)
expect(getDisplayText(editor)).toBe(dedent `
@@ -588,8 +920,6 @@ describe('TreeSitterLanguageMode', () => {
]
})
- const languageMode = new TreeSitterLanguageMode({buffer, grammar})
- buffer.setLanguageMode(languageMode)
buffer.setText(dedent `
@@ -597,7 +927,9 @@ describe('TreeSitterLanguageMode', () => {
`)
- await languageMode.reparsePromise
+ const languageMode = new TreeSitterLanguageMode({buffer, grammar})
+ buffer.setLanguageMode(languageMode)
+ await nextHighlightingUpdate(languageMode)
// Void elements have only one child
expect(editor.isFoldableAtBufferRow(1)).toBe(false)
@@ -611,7 +943,7 @@ describe('TreeSitterLanguageMode', () => {
})
describe('when folding a node that ends with a line break', () => {
- it('ends the fold at the end of the previous line', () => {
+ it('ends the fold at the end of the previous line', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, pythonGrammarPath, {
parser: 'tree-sitter-python',
folds: [
@@ -631,9 +963,9 @@ describe('TreeSitterLanguageMode', () => {
print 'c'
print 'd'
`)
- buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
- editor.screenLineForScreenRow(0)
+ buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
+ await nextHighlightingUpdate(buffer.getLanguageMode())
editor.foldBufferRow(0)
expect(getDisplayText(editor)).toBe(dedent `
@@ -645,19 +977,95 @@ describe('TreeSitterLanguageMode', () => {
`)
})
})
+
+ it('folds code in injected languages', async () => {
+ const htmlGrammar = new TreeSitterGrammar(atom.grammars, htmlGrammarPath, {
+ id: 'html',
+ parser: 'tree-sitter-html',
+ scopes: {},
+ folds: [{
+ type: ['element', 'raw_element'],
+ start: {index: 0},
+ end: {index: -1}
+ }],
+ injectionRegExp: 'html'
+ })
+
+ const jsGrammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
+ id: 'javascript',
+ parser: 'tree-sitter-javascript',
+ scopes: {},
+ folds: [{
+ type: ['template_string'],
+ start: {index: 0},
+ end: {index: -1},
+ },
+ {
+ start: {index: 0, type: '('},
+ end: {index: -1, type: ')'}
+ }],
+ injectionRegExp: 'javascript',
+ injectionPoints: [HTML_TEMPLATE_LITERAL_INJECTION_POINT]
+ })
+
+ atom.grammars.addGrammar(htmlGrammar)
+
+ buffer.setText(
+ `a = html \`
+
+ c\${def(
+ 1,
+ 2,
+ 3,
+ )}e\${f}g
+
+ \`
+ `
+ )
+ const languageMode = new TreeSitterLanguageMode({buffer, grammar: jsGrammar, grammars: atom.grammars})
+ buffer.setLanguageMode(languageMode)
+
+ await nextHighlightingUpdate(languageMode)
+ await nextHighlightingUpdate(languageMode)
+
+ editor.foldBufferRow(2)
+ expect(getDisplayText(editor)).toBe(
+ `a = html \`
+
+ c\${def(…)}e\${f}g
+
+ \`
+ `
+ )
+
+ editor.foldBufferRow(1)
+ expect(getDisplayText(editor)).toBe(
+ `a = html \`
+ …
+
+ \`
+ `
+ )
+
+ editor.foldBufferRow(0)
+ expect(getDisplayText(editor)).toBe(
+ `a = html \`…\`
+ `
+ )
+ })
})
describe('.scopeDescriptorForPosition', () => {
- it('returns a scope descriptor representing the given position in the syntax tree', () => {
+ it('returns a scope descriptor representing the given position in the syntax tree', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
id: 'javascript',
parser: 'tree-sitter-javascript'
})
buffer.setText('foo({bar: baz});')
- buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
- editor.screenLineForScreenRow(0)
+ buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
+ await nextHighlightingUpdate(buffer.getLanguageMode())
expect(editor.scopeDescriptorForBufferPosition([0, 6]).getScopesArray()).toEqual([
'javascript',
'program',
@@ -669,10 +1077,65 @@ describe('TreeSitterLanguageMode', () => {
'property_identifier'
])
})
+
+ it('includes nodes in injected syntax trees', async () => {
+ const jsGrammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
+ id: 'javascript',
+ parser: 'tree-sitter-javascript',
+ scopes: {},
+ injectionRegExp: 'javascript',
+ injectionPoints: [HTML_TEMPLATE_LITERAL_INJECTION_POINT]
+ })
+
+ const htmlGrammar = new TreeSitterGrammar(atom.grammars, htmlGrammarPath, {
+ id: 'html',
+ parser: 'tree-sitter-html',
+ scopes: {},
+ injectionRegExp: 'html',
+ injectionPoints: [SCRIPT_TAG_INJECTION_POINT]
+ })
+
+ atom.grammars.addGrammar(jsGrammar)
+ atom.grammars.addGrammar(htmlGrammar)
+
+ buffer.setText(`
+
+
+
+ `)
+
+ const languageMode = new TreeSitterLanguageMode({buffer, grammar: htmlGrammar, grammars: atom.grammars})
+ buffer.setLanguageMode(languageMode)
+ await nextHighlightingUpdate(languageMode)
+ await nextHighlightingUpdate(languageMode)
+ await nextHighlightingUpdate(languageMode)
+
+ const position = buffer.findSync('name').start
+ expect(languageMode.scopeDescriptorForPosition(position).getScopesArray()).toEqual([
+ 'html',
+ 'fragment',
+ 'element',
+ 'raw_element',
+ 'raw_text',
+ 'program',
+ 'expression_statement',
+ 'call_expression',
+ 'template_string',
+ 'fragment',
+ 'element',
+ 'template_substitution',
+ 'member_expression',
+ 'property_identifier'
+ ])
+ })
})
describe('TextEditor.selectLargerSyntaxNode and .selectSmallerSyntaxNode', () => {
- it('expands and contract the selection based on the syntax tree', () => {
+ it('expands and contracts the selection based on the syntax tree', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
scopes: {'program': 'source'}
@@ -684,9 +1147,9 @@ describe('TreeSitterLanguageMode', () => {
g()
}
`)
- buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
- editor.screenLineForScreenRow(0)
+ buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
+ await nextHighlightingUpdate(buffer.getLanguageMode())
editor.setCursorBufferPosition([1, 3])
editor.selectLargerSyntaxNode()
@@ -711,9 +1174,72 @@ describe('TreeSitterLanguageMode', () => {
editor.selectSmallerSyntaxNode()
expect(editor.getSelectedBufferRange()).toEqual([[1, 3], [1, 3]])
})
+
+ it('handles injected languages', async () => {
+ const jsGrammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
+ id: 'javascript',
+ parser: 'tree-sitter-javascript',
+ scopes: {
+ 'property_identifier': 'property',
+ 'call_expression > identifier': 'function',
+ 'template_string': 'string',
+ 'template_substitution > "${"': 'interpolation',
+ 'template_substitution > "}"': 'interpolation'
+ },
+ injectionRegExp: 'javascript',
+ injectionPoints: [HTML_TEMPLATE_LITERAL_INJECTION_POINT]
+ })
+
+ const htmlGrammar = new TreeSitterGrammar(atom.grammars, htmlGrammarPath, {
+ id: 'html',
+ parser: 'tree-sitter-html',
+ scopes: {
+ fragment: 'html',
+ tag_name: 'tag',
+ attribute_name: 'attr'
+ },
+ injectionRegExp: 'html'
+ })
+
+ atom.grammars.addGrammar(htmlGrammar)
+
+ buffer.setText('a = html ` c${def()}e${f}g `')
+ const languageMode = new TreeSitterLanguageMode({buffer, grammar: jsGrammar, grammars: atom.grammars})
+ buffer.setLanguageMode(languageMode)
+
+ await nextHighlightingUpdate(languageMode)
+ await nextHighlightingUpdate(languageMode)
+
+ editor.setCursorBufferPosition({row: 0, column: buffer.getText().indexOf('ef()')})
+ editor.selectLargerSyntaxNode()
+ expect(editor.getSelectedText()).toBe('def')
+ editor.selectLargerSyntaxNode()
+ expect(editor.getSelectedText()).toBe('def()')
+ editor.selectLargerSyntaxNode()
+ expect(editor.getSelectedText()).toBe('${def()}')
+ editor.selectLargerSyntaxNode()
+ expect(editor.getSelectedText()).toBe('c${def()}e${f}g')
+ editor.selectLargerSyntaxNode()
+ expect(editor.getSelectedText()).toBe('c${def()}e${f}g')
+ editor.selectLargerSyntaxNode()
+ expect(editor.getSelectedText()).toBe(' c${def()}e${f}g ')
+ editor.selectLargerSyntaxNode()
+ expect(editor.getSelectedText()).toBe('` c${def()}e${f}g `')
+ editor.selectLargerSyntaxNode()
+ expect(editor.getSelectedText()).toBe('html ` c${def()}e${f}g `')
+ })
})
})
+function nextHighlightingUpdate (languageMode) {
+ return new Promise(resolve => {
+ const subscription = languageMode.onDidChangeHighlighting(() => {
+ subscription.dispose()
+ resolve()
+ })
+ })
+}
+
function getDisplayText (editor) {
return editor.displayLayer.getText()
}
@@ -740,7 +1266,7 @@ function expectTokensToEqual (editor, expectedTokenLines) {
text,
scopes: scopes.map(scope => scope
.split(' ')
- .map(className => className.slice('syntax--'.length))
+ .map(className => className.replace('syntax--', ''))
.join(' '))
}))
}
@@ -760,3 +1286,21 @@ function expectTokensToEqual (editor, expectedTokenLines) {
// due to subsequent edits can be tested.
editor.displayLayer.getScreenLines(0, Infinity)
}
+
+const HTML_TEMPLATE_LITERAL_INJECTION_POINT = {
+ type: 'call_expression',
+ language (node) {
+ if (node.lastChild.type === 'template_string' && node.firstChild.type === 'identifier') {
+ return node.firstChild.text
+ }
+ },
+ content (node) {
+ return node.lastChild
+ }
+}
+
+const SCRIPT_TAG_INJECTION_POINT = {
+ type: 'raw_element',
+ language () { return 'javascript' },
+ content (node) { return node.child(1) }
+}
diff --git a/src/grammar-registry.js b/src/grammar-registry.js
index a2a5917da..101a38007 100644
--- a/src/grammar-registry.js
+++ b/src/grammar-registry.js
@@ -169,7 +169,7 @@ class GrammarRegistry {
languageModeForGrammarAndBuffer (grammar, buffer) {
if (grammar instanceof TreeSitterGrammar) {
- return new TreeSitterLanguageMode({grammar, buffer, config: this.config})
+ return new TreeSitterLanguageMode({grammar, buffer, config: this.config, grammars: this})
} else {
return new TextMateLanguageMode({grammar, buffer, config: this.config})
}
@@ -291,8 +291,9 @@ class GrammarRegistry {
forEachGrammar (callback) {
this.textmateRegistry.grammars.forEach(callback)
- for (let grammarId in this.treeSitterGrammarsById) {
- callback(this.treeSitterGrammarsById[grammarId])
+ for (const grammarId in this.treeSitterGrammarsById) {
+ const grammar = this.treeSitterGrammarsById[grammarId]
+ if (grammar.id) callback(grammar)
}
}
@@ -347,26 +348,23 @@ class GrammarRegistry {
this.grammarScoresByBuffer.forEach((score, buffer) => {
const languageMode = buffer.getLanguageMode()
- if (grammar.injectionSelector) {
- if (languageMode.hasTokenForSelector(grammar.injectionSelector)) {
- languageMode.retokenizeLines()
- }
- return
- }
-
const languageOverride = this.languageOverridesByBufferId.get(buffer.id)
if ((grammar.id === buffer.getLanguageMode().getLanguageId() ||
grammar.id === languageOverride)) {
buffer.setLanguageMode(this.languageModeForGrammarAndBuffer(grammar, buffer))
+ return
} else if (!languageOverride) {
const score = this.getGrammarScore(grammar, buffer.getPath(), getGrammarSelectionContent(buffer))
const currentScore = this.grammarScoresByBuffer.get(buffer)
if (currentScore == null || score > currentScore) {
buffer.setLanguageMode(this.languageModeForGrammarAndBuffer(grammar, buffer))
this.grammarScoresByBuffer.set(buffer, score)
+ return
}
}
+
+ languageMode.updateForInjection(grammar)
})
}
@@ -391,6 +389,32 @@ class GrammarRegistry {
return this.textmateRegistry.onDidUpdateGrammar(callback)
}
+ // Experimental: Specify a type of syntax node that may embed other languages.
+ //
+ // * `grammarId` The {String} id of the parent language
+ // * `injectionPoint` An {Object} with the following keys:
+ // * `type` The {String} type of syntax node that may embed other languages
+ // * `language` A {Function} that is called with syntax nodes of the specified `type` and
+ // returns a {String} that will be tested against other grammars' `injectionRegExp` in
+ // order to determine what language should be embedded.
+ // * `content` A {Function} that is called with syntax nodes of the specified `type` and
+ // returns another syntax node or array of syntax nodes that contain the embedded source code.
+ addInjectionPoint (grammarId, injectionPoint) {
+ const grammar = this.treeSitterGrammarsById[grammarId]
+ if (grammar) {
+ grammar.injectionPoints.push(injectionPoint)
+ } else {
+ this.treeSitterGrammarsById[grammarId] = {
+ injectionPoints: [injectionPoint]
+ }
+ }
+ return new Disposable(() => {
+ const grammar = this.treeSitterGrammarsById[grammarId]
+ const index = grammar.injectionPoints.indexOf(injectionPoint)
+ if (index !== -1) grammar.injectionPoints.splice(index, 1)
+ })
+ }
+
get nullGrammar () {
return this.textmateRegistry.nullGrammar
}
@@ -409,12 +433,14 @@ class GrammarRegistry {
addGrammar (grammar) {
if (grammar instanceof TreeSitterGrammar) {
+ const existingParams = this.treeSitterGrammarsById[grammar.id] || {}
this.treeSitterGrammarsById[grammar.id] = grammar
if (grammar.legacyScopeName) {
this.config.setLegacyScopeAliasForNewScope(grammar.id, grammar.legacyScopeName)
this.textMateScopeNamesByTreeSitterLanguageId.set(grammar.id, grammar.legacyScopeName)
this.treeSitterLanguageIdsByTextMateScopeName.set(grammar.legacyScopeName, grammar.id)
}
+ if (existingParams.injectionPoints) grammar.injectionPoints.push(...existingParams.injectionPoints)
this.grammarAddedOrUpdated(grammar)
return new Disposable(() => this.removeGrammar(grammar))
} else {
@@ -515,6 +541,15 @@ class GrammarRegistry {
return this.textmateRegistry.scopeForId(id)
}
+ treeSitterGrammarForLanguageString (languageString) {
+ for (const id in this.treeSitterGrammarsById) {
+ const grammar = this.treeSitterGrammarsById[id]
+ if (grammar.injectionRegExp && grammar.injectionRegExp.test(languageString)) {
+ return grammar
+ }
+ }
+ }
+
normalizeLanguageId (languageId) {
if (this.config.get('core.useTreeSitterParsers')) {
return this.treeSitterLanguageIdsByTextMateScopeName.get(languageId) || languageId
diff --git a/src/initialize-application-window.coffee b/src/initialize-application-window.coffee
index f8f670cf5..e3e24eb87 100644
--- a/src/initialize-application-window.coffee
+++ b/src/initialize-application-window.coffee
@@ -36,6 +36,8 @@ if global.isGeneratingSnapshot
require('image-view')
require('incompatible-packages')
require('keybinding-resolver')
+ require('language-html')
+ require('language-javascript')
require('line-ending-selector')
require('link')
require('markdown-preview')
diff --git a/src/project.js b/src/project.js
index 8c98224d0..4e51efcf8 100644
--- a/src/project.js
+++ b/src/project.js
@@ -685,9 +685,6 @@ class Project extends Model {
}
this.grammarRegistry.autoAssignLanguageMode(buffer)
- if (buffer.languageMode.initialize) {
- await buffer.languageMode.initialize()
- }
this.addBuffer(buffer)
return buffer
diff --git a/src/test.ejs b/src/test.ejs
new file mode 100644
index 000000000..7b93c31b3
--- /dev/null
+++ b/src/test.ejs
@@ -0,0 +1,9 @@
+
+
+<% if something() { %>
+
+ <%= html `ok how about this` %>
+
+<% } %>
+
+
diff --git a/src/text-mate-language-mode.js b/src/text-mate-language-mode.js
index 152636ab7..9abe55ecb 100644
--- a/src/text-mate-language-mode.js
+++ b/src/text-mate-language-mode.js
@@ -235,15 +235,18 @@ class TextMateLanguageMode {
return this.buffer.getTextInRange([[0, 0], [10, 0]])
}
- hasTokenForSelector (selector) {
+ updateForInjection (grammar) {
+ if (!grammar.injectionSelector) return
for (const tokenizedLine of this.tokenizedLines) {
if (tokenizedLine) {
for (let token of tokenizedLine.tokens) {
- if (selector.matches(token.scopes)) return true
+ if (grammar.injectionSelector.matches(token.scopes)) {
+ this.retokenizeLines()
+ return
+ }
}
}
}
- return false
}
retokenizeLines () {
diff --git a/src/tree-sitter-grammar.js b/src/tree-sitter-grammar.js
index d00344fb1..acea24213 100644
--- a/src/tree-sitter-grammar.js
+++ b/src/tree-sitter-grammar.js
@@ -10,6 +10,7 @@ class TreeSitterGrammar {
this.name = params.name
this.legacyScopeName = params.legacyScopeName
if (params.contentRegExp) this.contentRegExp = new RegExp(params.contentRegExp)
+ if (params.injectionRegExp) this.injectionRegExp = new RegExp(params.injectionRegExp)
this.folds = params.folds || []
@@ -28,6 +29,7 @@ class TreeSitterGrammar {
this.scopeMap = new SyntaxScopeMap(scopeSelectors)
this.fileTypes = params.fileTypes
+ this.injectionPoints = params.injectionPoints || []
// TODO - When we upgrade to a new enough version of node, use `require.resolve`
// with the new `paths` option instead of this private API.
diff --git a/src/tree-sitter-language-mode.js b/src/tree-sitter-language-mode.js
index 5d8e743ed..7d0377f28 100644
--- a/src/tree-sitter-language-mode.js
+++ b/src/tree-sitter-language-mode.js
@@ -1,119 +1,115 @@
const Parser = require('tree-sitter')
const {Point, Range} = require('text-buffer')
+const {Patch} = require('superstring')
const {Emitter, Disposable} = require('event-kit')
const ScopeDescriptor = require('./scope-descriptor')
const TokenizedLine = require('./tokenized-line')
const TextMateLanguageMode = require('./text-mate-language-mode')
let nextId = 0
+const MAX_RANGE = new Range(Point.ZERO, Point.INFINITY).freeze()
+const PARSER_POOL = []
-module.exports =
class TreeSitterLanguageMode {
- constructor ({buffer, grammar, config}) {
+ static _patchSyntaxNode () {
+ if (!Parser.SyntaxNode.prototype.hasOwnProperty('text')) {
+ Object.defineProperty(Parser.SyntaxNode.prototype, 'text', {
+ get () {
+ return this.tree.buffer.getTextInRange(new Range(this.startPosition, this.endPosition))
+ }
+ })
+ }
+ }
+
+ constructor ({buffer, grammar, config, grammars}) {
+ TreeSitterLanguageMode._patchSyntaxNode()
this.id = nextId++
this.buffer = buffer
this.grammar = grammar
this.config = config
+ this.grammarRegistry = grammars
this.parser = new Parser()
- this.parser.setLanguage(grammar.languageModule)
- this.tree = null
+ this.rootLanguageLayer = new LanguageLayer(this, grammar)
+ this.injectionsMarkerLayer = buffer.addMarkerLayer()
+
this.rootScopeDescriptor = new ScopeDescriptor({scopes: [this.grammar.id]})
this.emitter = new Emitter()
this.isFoldableCache = []
this.hasQueuedParse = false
- this.changeListsSinceCurrentParse = []
- this.subscription = this.buffer.onDidChangeText(async ({changes}) => {
- if (this.reparsePromise) {
- this.changeListsSinceCurrentParse.push(changes)
- } else {
- this.reparsePromise = this.reparse()
+
+ this.grammarForLanguageString = this.grammarForLanguageString.bind(this)
+ this.emitRangeUpdate = this.emitRangeUpdate.bind(this)
+
+ this.subscription = this.buffer.onDidChangeText(({changes}) => {
+ for (let i = changes.length - 1; i >= 0; i--) {
+ const {oldRange, newRange} = changes[i]
+ const startRow = oldRange.start.row
+ const oldEndRow = oldRange.end.row
+ const newEndRow = newRange.end.row
+ this.isFoldableCache.splice(
+ startRow,
+ oldEndRow - startRow,
+ ...new Array(newEndRow - startRow)
+ )
}
+
+ this.rootLanguageLayer.update(null)
})
+ this.rootLanguageLayer.update(null)
+
// TODO: Remove this once TreeSitterLanguageMode implements its own auto-indentation system. This
// is temporarily needed in order to delegate to the TextMateLanguageMode's auto-indent system.
this.regexesByPattern = {}
}
destroy () {
+ this.injectionsMarkerLayer.destroy()
this.subscription.dispose()
- this.tree = null
+ this.rootLanguageLayer = null
this.parser = null
}
- async initialize () {
- this.tree = await this.parser.parseTextBuffer(this.buffer.buffer)
- }
-
- ensureParseTree () {
- if (!this.tree) {
- this.tree = this.parser.parseTextBufferSync(this.buffer.buffer)
- }
- }
-
getLanguageId () {
return this.grammar.id
}
bufferDidChange (change) {
- this.ensureParseTree()
- const {oldRange, newRange} = change
- const startRow = oldRange.start.row
- const oldEndRow = oldRange.end.row
- const newEndRow = newRange.end.row
- this.isFoldableCache.splice(startRow, oldEndRow - startRow, ...new Array(newEndRow - startRow))
- this.tree.edit(this.treeEditForBufferChange(change))
+ this.rootLanguageLayer.handleTextChange(change)
+ for (const marker of this.injectionsMarkerLayer.getMarkers()) {
+ marker.languageLayer.handleTextChange(change)
+ }
+ }
+
+ async parse (language, oldTree, ranges) {
+ const parser = PARSER_POOL.pop() || new Parser()
+ parser.setLanguage(language)
+ const newTree = await parser.parseTextBuffer(this.buffer.buffer, oldTree, {
+ syncOperationLimit: 1000,
+ includedRanges: ranges
+ })
+ PARSER_POOL.push(parser)
+ return newTree
+ }
+
+ get tree () {
+ return this.rootLanguageLayer.tree
+ }
+
+ updateForInjection (grammar) {
+ this.rootLanguageLayer.updateInjections(grammar)
}
/*
Section - Highlighting
*/
- treeEditForBufferChange ({oldRange, newRange, oldText, newText}) {
- const startIndex = this.buffer.characterIndexForPosition(oldRange.start)
- return {
- startIndex,
- oldEndIndex: startIndex + oldText.length,
- newEndIndex: startIndex + newText.length,
- startPosition: oldRange.start,
- oldEndPosition: oldRange.end,
- newEndPosition: newRange.end
- }
- }
-
- async reparse () {
- const tree = await this.parser.parseTextBuffer(this.buffer.buffer, this.tree, {
- syncOperationLimit: 1000
- })
- const invalidatedRanges = this.tree.getChangedRanges(tree)
-
- for (let i = 0, n = invalidatedRanges.length; i < n; i++) {
- const range = invalidatedRanges[i]
- const startRow = range.start.row
- const endRow = range.end.row
- for (let row = startRow; row < endRow; row++) {
- this.isFoldableCache[row] = undefined
- }
- this.emitter.emit('did-change-highlighting', range)
- }
-
- this.tree = tree
- if (this.changeListsSinceCurrentParse.length > 0) {
- for (const changeList of this.changeListsSinceCurrentParse) {
- for (let i = changeList.length - 1; i >= 0; i--) {
- this.tree.edit(this.treeEditForBufferChange(changeList[i]))
- }
- }
- this.changeListsSinceCurrentParse.length = 0
- this.reparsePromise = this.reparse()
- } else {
- this.reparsePromise = null
- }
- }
-
buildHighlightIterator () {
- this.ensureParseTree()
- return new TreeSitterHighlightIterator(this, this.tree.walk())
+ const layerIterators = [
+ this.rootLanguageLayer.buildHighlightIterator(),
+ ...this.injectionsMarkerLayer.getMarkers().map(m => m.languageLayer.buildHighlightIterator())
+ ]
+ return new HighlightIterator(this, layerIterators)
}
onDidChangeHighlighting (callback) {
@@ -189,14 +185,17 @@ class TreeSitterLanguageMode {
return this.getFoldableRangesAtIndentLevel(null)
}
+ /**
+ * TODO: Make this method generate folds for nested languages (currently,
+ * folds are only generated for the root language layer).
+ */
getFoldableRangesAtIndentLevel (goalLevel) {
- this.ensureParseTree()
let result = []
let stack = [{node: this.tree.rootNode, level: 0}]
while (stack.length > 0) {
const {node, level} = stack.pop()
- const range = this.getFoldableRangeForNode(node)
+ const range = this.getFoldableRangeForNode(node, this.grammar)
if (range) {
if (goalLevel == null || level === goalLevel) {
let updatedExistingRange = false
@@ -236,25 +235,49 @@ class TreeSitterLanguageMode {
}
getFoldableRangeContainingPoint (point, tabLength, existenceOnly = false) {
- this.ensureParseTree()
- let node = this.tree.rootNode.descendantForPosition(this.buffer.clipPosition(point))
- while (node) {
- if (existenceOnly && node.startPosition.row < point.row) break
- if (node.endPosition.row > point.row) {
- const range = this.getFoldableRangeForNode(node, existenceOnly)
- if (range) return range
+ if (!this.tree) return null
+
+ let smallestRange
+ this._forEachTreeWithRange(new Range(point, point), (tree, grammar) => {
+ let node = tree.rootNode.descendantForPosition(this.buffer.clipPosition(point))
+ while (node) {
+ if (existenceOnly && node.startPosition.row < point.row) return
+ if (node.endPosition.row > point.row) {
+ const range = this.getFoldableRangeForNode(node, grammar)
+ if (range && rangeIsSmaller(range, smallestRange)) {
+ smallestRange = range
+ return
+ }
+ }
+ node = node.parent
}
- node = node.parent
+ })
+
+ return existenceOnly
+ ? smallestRange && smallestRange.start.row === point.row
+ : smallestRange
+ }
+
+ _forEachTreeWithRange (range, callback) {
+ callback(this.rootLanguageLayer.tree, this.rootLanguageLayer.grammar)
+
+ const injectionMarkers = this.injectionsMarkerLayer.findMarkers({
+ intersectsRange: range
+ })
+
+ for (const injectionMarker of injectionMarkers) {
+ const {tree, grammar} = injectionMarker.languageLayer
+ if (tree) callback(tree, grammar)
}
}
- getFoldableRangeForNode (node, existenceOnly) {
+ getFoldableRangeForNode (node, grammar, existenceOnly) {
const {children, type: nodeType} = node
const childCount = children.length
let childTypes
- for (var i = 0, {length} = this.grammar.folds; i < length; i++) {
- const foldEntry = this.grammar.folds[i]
+ for (var i = 0, {length} = grammar.folds; i < length; i++) {
+ const foldEntry = grammar.folds[i]
if (foldEntry.type) {
if (typeof foldEntry.type === 'string') {
@@ -322,17 +345,24 @@ class TreeSitterLanguageMode {
}
/*
- Syntax Tree APIs
+ Section - Syntax Tree APIs
*/
getRangeForSyntaxNodeContainingRange (range) {
const startIndex = this.buffer.characterIndexForPosition(range.start)
const endIndex = this.buffer.characterIndexForPosition(range.end)
- let node = this.tree.rootNode.descendantForIndex(startIndex, endIndex - 1)
- while (node && node.startIndex === startIndex && node.endIndex === endIndex) {
- node = node.parent
- }
- if (node) return new Range(node.startPosition, node.endPosition)
+ const searchEndIndex = Math.max(0, endIndex - 1)
+
+ let smallestNode
+ this._forEachTreeWithRange(range, tree => {
+ let node = tree.rootNode.descendantForIndex(startIndex, searchEndIndex)
+ while (node && !nodeContainsIndices(node, startIndex, endIndex)) {
+ node = node.parent
+ }
+ if (nodeIsSmaller(node, smallestNode)) smallestNode = node
+ })
+
+ if (smallestNode) return rangeForNode(smallestNode)
}
bufferRangeForScopeAtPosition (position) {
@@ -358,53 +388,348 @@ class TreeSitterLanguageMode {
}
scopeDescriptorForPosition (point) {
- this.ensureParseTree()
+ if (!this.tree) return this.rootScopeDescriptor
point = Point.fromObject(point)
- let node = this.tree.rootNode.descendantForPosition(point)
- // Don't include anonymous token types like '(' because they prevent scope chains
- // from being parsed as CSS selectors by the `slick` parser. Other css selector
- // parsers like `postcss-selector-parser` do allow arbitrary quoted strings in
- // selectors.
- if (!node.isNamed) node = node.parent
+ const iterators = []
+ this._forEachTreeWithRange(new Range(point, point), tree => {
+ const rootStartIndex = tree.rootNode.startIndex
+ let node = tree.rootNode.descendantForPosition(point)
- const result = []
- while (node) {
- result.push(node.type)
- node = node.parent
+ // Don't include anonymous token types like '(' because they prevent scope chains
+ // from being parsed as CSS selectors by the `slick` parser. Other css selector
+ // parsers like `postcss-selector-parser` do allow arbitrary quoted strings in
+ // selectors.
+ if (!node.isNamed) node = node.parent
+ iterators.push({node, rootStartIndex})
+ })
+
+ iterators.sort(compareScopeDescriptorIterators)
+
+ const scopes = []
+ for (;;) {
+ const {length} = iterators
+ if (!length) break
+ const iterator = iterators[length - 1]
+ scopes.push(iterator.node.type)
+ iterator.node = iterator.node.parent
+ if (iterator.node) {
+ let i = length - 1
+ while (i > 0 && compareScopeDescriptorIterators(iterator, iterators[i - 1]) < 0) i--
+ if (i < length - 1) iterators.splice(i, 0, iterators.pop())
+ } else {
+ iterators.pop()
+ }
}
- result.push(this.grammar.id)
- return new ScopeDescriptor({scopes: result.reverse()})
- }
- hasTokenForSelector (scopeSelector) {
- return false
+ scopes.push(this.grammar.id)
+ return new ScopeDescriptor({scopes: scopes.reverse()})
}
getGrammar () {
return this.grammar
}
+
+ /*
+ Section - Private
+ */
+
+ grammarForLanguageString (languageString) {
+ return this.grammarRegistry.treeSitterGrammarForLanguageString(languageString)
+ }
+
+ emitRangeUpdate (range) {
+ const startRow = range.start.row
+ const endRow = range.end.row
+ for (let row = startRow; row < endRow; row++) {
+ this.isFoldableCache[row] = undefined
+ }
+ this.emitter.emit('did-change-highlighting', range)
+ }
}
-class TreeSitterHighlightIterator {
- constructor (languageMode, treeCursor) {
+class LanguageLayer {
+ constructor (languageMode, grammar, contentChildTypes) {
this.languageMode = languageMode
+ this.grammar = grammar
+ this.tree = null
+ this.currentParsePromise = null
+ this.patchSinceCurrentParseStarted = null
+ this.contentChildTypes = contentChildTypes
+ }
+
+ buildHighlightIterator () {
+ if (this.tree) {
+ return new LayerHighlightIterator(this, this.tree.walk())
+ } else {
+ return new NullHighlightIterator()
+ }
+ }
+
+ handleTextChange ({oldRange, newRange, oldText, newText}) {
+ if (this.tree) {
+ this.tree.edit(this._treeEditForBufferChange(
+ oldRange.start, oldRange.end, newRange.end, oldText, newText
+ ))
+
+ if (this.editedRange) {
+ if (newRange.start.isLessThan(this.editedRange.start)) {
+ this.editedRange.start = newRange.start
+ }
+ if (oldRange.end.isLessThan(this.editedRange.end)) {
+ this.editedRange.end = newRange.end.traverse(this.editedRange.end.traversalFrom(oldRange.end))
+ } else {
+ this.editedRange.end = newRange.end
+ }
+ } else {
+ this.editedRange = newRange.copy()
+ }
+ }
+
+ if (this.patchSinceCurrentParseStarted) {
+ this.patchSinceCurrentParseStarted.splice(
+ oldRange.start,
+ oldRange.end,
+ newRange.end,
+ oldText,
+ newText
+ )
+ }
+ }
+
+ destroy () {
+ for (const marker of this.languageMode.injectionsMarkerLayer.getMarkers()) {
+ if (marker.parentLanguageLayer === this) {
+ marker.languageLayer.destroy()
+ marker.destroy()
+ }
+ }
+ }
+
+ async update (nodeRangeSet) {
+ if (!this.currentParsePromise) {
+ do {
+ this.currentParsePromise = this._performUpdate(nodeRangeSet)
+ await this.currentParsePromise
+ } while (this.tree && this.tree.rootNode.hasChanges())
+ this.currentParsePromise = null
+ }
+ }
+
+ updateInjections (grammar) {
+ if (grammar.injectionRegExp) {
+ if (!this.currentParsePromise) this.currentParsePromise = Promise.resolve()
+ this.currentParsePromise = this.currentParsePromise.then(async () => {
+ await this._populateInjections(MAX_RANGE, null)
+ this.currentParsePromise = null
+ })
+ }
+ }
+
+ async _performUpdate (nodeRangeSet) {
+ let includedRanges = null
+ if (nodeRangeSet) {
+ includedRanges = nodeRangeSet.getRanges()
+ if (includedRanges.length === 0) {
+ this.tree = null
+ return
+ }
+ }
+
+ let affectedRange = this.editedRange
+ this.editedRange = null
+
+ this.patchSinceCurrentParseStarted = new Patch()
+ const tree = await this.languageMode.parse(
+ this.grammar.languageModule,
+ this.tree,
+ includedRanges
+ )
+ tree.buffer = this.languageMode.buffer
+
+ const changes = this.patchSinceCurrentParseStarted.getChanges()
+ this.patchSinceCurrentParseStarted = null
+ for (let i = changes.length - 1; i >= 0; i--) {
+ const {oldStart, oldEnd, newEnd, oldText, newText} = changes[i]
+ tree.edit(this._treeEditForBufferChange(
+ oldStart, oldEnd, newEnd, oldText, newText
+ ))
+ }
+
+ if (this.tree) {
+ const rangesWithSyntaxChanges = this.tree.getChangedRanges(tree)
+ this.tree = tree
+
+ if (!affectedRange) return
+ if (rangesWithSyntaxChanges.length > 0) {
+ for (const range of rangesWithSyntaxChanges) {
+ this.languageMode.emitRangeUpdate(rangeForNode(range))
+ }
+
+ affectedRange = affectedRange.union(new Range(
+ rangesWithSyntaxChanges[0].startPosition,
+ last(rangesWithSyntaxChanges).endPosition
+ ))
+ } else {
+ this.languageMode.emitRangeUpdate(affectedRange)
+ }
+ } else {
+ this.tree = tree
+ this.languageMode.emitRangeUpdate(rangeForNode(tree.rootNode))
+ if (includedRanges) {
+ affectedRange = new Range(includedRanges[0].startPosition, last(includedRanges).endPosition)
+ } else {
+ affectedRange = MAX_RANGE
+ }
+ }
+
+ await this._populateInjections(affectedRange, nodeRangeSet)
+ }
+
+ _populateInjections (range, nodeRangeSet) {
+ const {injectionsMarkerLayer, grammarForLanguageString} = this.languageMode
+
+ const existingInjectionMarkers = injectionsMarkerLayer
+ .findMarkers({intersectsRange: range})
+ .filter(marker => marker.parentLanguageLayer === this)
+
+ if (existingInjectionMarkers.length > 0) {
+ range = range.union(new Range(
+ existingInjectionMarkers[0].getRange().start,
+ last(existingInjectionMarkers).getRange().end
+ ))
+ }
+
+ const markersToUpdate = new Map()
+ for (const injectionPoint of this.grammar.injectionPoints) {
+ const nodes = this.tree.rootNode.descendantsOfType(
+ injectionPoint.type,
+ range.start,
+ range.end
+ )
+
+ for (const node of nodes) {
+ const languageName = injectionPoint.language(node)
+ if (!languageName) continue
+
+ const grammar = grammarForLanguageString(languageName)
+ if (!grammar) continue
+
+ const contentNodes = injectionPoint.content(node)
+ if (!contentNodes) continue
+
+ const injectionNodes = [].concat(contentNodes)
+ if (!injectionNodes.length) continue
+
+ const injectionRange = rangeForNode(node)
+ let marker = existingInjectionMarkers.find(m =>
+ m.getRange().isEqual(injectionRange) &&
+ m.languageLayer.grammar === grammar
+ )
+ if (!marker) {
+ marker = injectionsMarkerLayer.markRange(injectionRange)
+ marker.languageLayer = new LanguageLayer(this.languageMode, grammar, injectionPoint.contentChildTypes)
+ marker.parentLanguageLayer = this
+ }
+
+ markersToUpdate.set(marker, new NodeRangeSet(nodeRangeSet, injectionNodes))
+ }
+ }
+
+ for (const marker of existingInjectionMarkers) {
+ if (!markersToUpdate.has(marker)) {
+ marker.languageLayer.destroy()
+ this.languageMode.emitRangeUpdate(marker.getRange())
+ marker.destroy()
+ }
+ }
+
+ const promises = []
+ for (const [marker, nodeRangeSet] of markersToUpdate) {
+ promises.push(marker.languageLayer.update(nodeRangeSet))
+ }
+ return Promise.all(promises)
+ }
+
+ _treeEditForBufferChange (start, oldEnd, newEnd, oldText, newText) {
+ const startIndex = this.languageMode.buffer.characterIndexForPosition(start)
+ return {
+ startIndex,
+ oldEndIndex: startIndex + oldText.length,
+ newEndIndex: startIndex + newText.length,
+ startPosition: start,
+ oldEndPosition: oldEnd,
+ newEndPosition: newEnd
+ }
+ }
+}
+
+class HighlightIterator {
+ constructor (languageMode, iterators) {
+ this.languageMode = languageMode
+ this.iterators = iterators.sort((a, b) => b.getIndex() - a.getIndex())
+ }
+
+ seek (targetPosition) {
+ const containingTags = []
+ const containingTagStartIndices = []
+ const targetIndex = this.languageMode.buffer.characterIndexForPosition(targetPosition)
+ for (let i = this.iterators.length - 1; i >= 0; i--) {
+ this.iterators[i].seek(targetIndex, containingTags, containingTagStartIndices)
+ }
+ this.iterators.sort((a, b) => b.getIndex() - a.getIndex())
+ return containingTags
+ }
+
+ moveToSuccessor () {
+ const lastIndex = this.iterators.length - 1
+ const leader = this.iterators[lastIndex]
+ leader.moveToSuccessor()
+ const leaderCharIndex = leader.getIndex()
+ let i = lastIndex
+ while (i > 0 && this.iterators[i - 1].getIndex() < leaderCharIndex) i--
+ if (i < lastIndex) this.iterators.splice(i, 0, this.iterators.pop())
+ }
+
+ getPosition () {
+ return last(this.iterators).getPosition()
+ }
+
+ getCloseScopeIds () {
+ return last(this.iterators).getCloseScopeIds()
+ }
+
+ getOpenScopeIds () {
+ return last(this.iterators).getOpenScopeIds()
+ }
+
+ logState () {
+ const iterator = last(this.iterators)
+ if (iterator.treeCursor) {
+ console.log(
+ iterator.getPosition(),
+ iterator.treeCursor.nodeType,
+ new Range(
+ iterator.languageLayer.tree.rootNode.startPosition,
+ iterator.languageLayer.tree.rootNode.endPosition
+ ).toString()
+ )
+ }
+ }
+}
+
+class LayerHighlightIterator {
+ constructor (languageLayer, treeCursor) {
+ this.languageLayer = languageLayer
this.treeCursor = treeCursor
+ this.atEnd = false
// In order to determine which selectors match its current node, the iterator maintains
// a list of the current node's ancestors. Because the selectors can use the `:nth-child`
// pseudo-class, each node's child index is also stored.
this.containingNodeTypes = []
this.containingNodeChildIndices = []
-
- // Conceptually, the iterator represents a single position in the text. It stores this
- // position both as a character index and as a `Point`. This position corresponds to a
- // leaf node of the syntax tree, which either contains or follows the iterator's
- // textual position. The `treeCursor` property points at that leaf node, and
- // `currentChildIndex` represents the child index of that leaf node within its parent.
- this.currentIndex = null
- this.currentPosition = null
- this.currentChildIndex = null
+ this.containingNodeEndIndices = []
// At any given position, the iterator exposes the list of class names that should be
// *ended* at its current position and the list of class names that should be *started*
@@ -413,113 +738,159 @@ class TreeSitterHighlightIterator {
this.openTags = []
}
- seek (targetPosition) {
+ seek (targetIndex, containingTags, containingTagStartIndices) {
while (this.treeCursor.gotoParent()) {}
- const containingTags = []
-
+ this.done = false
+ this.atEnd = true
this.closeTags.length = 0
this.openTags.length = 0
this.containingNodeTypes.length = 0
this.containingNodeChildIndices.length = 0
- this.currentPosition = targetPosition
- this.currentIndex = this.languageMode.buffer.characterIndexForPosition(targetPosition)
+ this.containingNodeEndIndices.length = 0
- // Descend from the root of the tree to the smallest node that spans the given position.
- // Keep track of any nodes along the way that are associated with syntax highlighting
- // tags. These tags must be returned.
- var childIndex = -1
- var nodeContainsTarget = true
+ const containingTagEndIndices = []
+
+ if (targetIndex >= this.treeCursor.endIndex) {
+ this.done = true
+ return
+ }
+
+ let childIndex = -1
for (;;) {
- this.currentChildIndex = childIndex
- if (!nodeContainsTarget) break
this.containingNodeTypes.push(this.treeCursor.nodeType)
this.containingNodeChildIndices.push(childIndex)
+ this.containingNodeEndIndices.push(this.treeCursor.endIndex)
const scopeName = this.currentScopeName()
if (scopeName) {
- const id = this.languageMode.grammar.idForScope(scopeName)
- if (this.currentIndex === this.treeCursor.startIndex) {
- this.openTags.push(id)
+ const id = this.idForScope(scopeName)
+ if (this.treeCursor.startIndex < targetIndex) {
+ insertContainingTag(id, this.treeCursor.startIndex, containingTags, containingTagStartIndices)
+ containingTagEndIndices.push(this.treeCursor.endIndex)
} else {
- containingTags.push(id)
+ this.atEnd = false
+ this.openTags.push(id)
+ while (this.treeCursor.gotoFirstChild()) {
+ this.containingNodeTypes.push(this.treeCursor.nodeType)
+ this.containingNodeChildIndices.push(0)
+ const scopeName = this.currentScopeName()
+ if (scopeName) {
+ this.openTags.push(this.idForScope(scopeName))
+ }
+ }
+ break
}
}
- const nextChildIndex = this.treeCursor.gotoFirstChildForIndex(this.currentIndex)
- if (nextChildIndex == null) break
- if (this.treeCursor.startIndex > this.currentIndex) nodeContainsTarget = false
- childIndex = nextChildIndex
+ childIndex = this.treeCursor.gotoFirstChildForIndex(targetIndex)
+ if (childIndex === null) break
+ if (this.treeCursor.startIndex >= targetIndex) this.atEnd = false
+ }
+
+ if (this.atEnd) {
+ const currentIndex = this.treeCursor.endIndex
+ for (let i = 0, {length} = containingTags; i < length; i++) {
+ if (containingTagEndIndices[i] === currentIndex) {
+ this.closeTags.push(containingTags[i])
+ }
+ }
}
return containingTags
}
moveToSuccessor () {
+ let didMove = false
this.closeTags.length = 0
this.openTags.length = 0
- // Step forward through the leaves of the tree to find the next place where one or more
- // syntax highlighting tags begin, end, or both.
- do {
- // If the iterator is before the beginning of the current node, advance it to the
- // beginning of then node and then walk down into the node's children, marking
- // open tags as needed.
- if (this.currentIndex < this.treeCursor.startIndex) {
- this.currentIndex = this.treeCursor.startIndex
- this.currentPosition = this.treeCursor.startPosition
- this.pushOpenTag()
- this.descendLeft()
+ if (this.done) return
- // If the iterator is within the current node, advance it to the end of the node
- // and then walk up the tree until the next sibling is found, marking close tags
- // as needed.
- //
- } else if (this.currentIndex < this.treeCursor.endIndex) {
- /* eslint-disable no-labels */
- ascendingLoop:
- do {
- this.currentIndex = this.treeCursor.endIndex
- this.currentPosition = this.treeCursor.endPosition
- this.pushCloseTag()
+ while (true) {
+ if (this.atEnd) {
+ if (this.treeCursor.gotoNextSibling()) {
+ didMove = true
+ this.atEnd = false
+ const depth = this.containingNodeTypes.length
+ this.containingNodeTypes[depth - 1] = this.treeCursor.nodeType
+ this.containingNodeChildIndices[depth - 1]++
+ this.containingNodeEndIndices[depth - 1] = this.treeCursor.endIndex
- // Stop walking upward when we reach a node with a next sibling.
- while (this.treeCursor.gotoNextSibling()) {
- this.currentChildIndex++
-
- // If the next sibling has a size of zero (e.g. something like an `automatic_semicolon`,
- // an `indent`, or a `MISSING` node inserted by the parser during error recovery),
- // then skip it. These nodes play no role in syntax highlighting.
- if (this.treeCursor.endIndex === this.currentIndex) continue
-
- // If the next sibling starts right at the end of the current node (i.e. there is
- // no whitespace in between), then before returning, also mark any open tags associated
- // with this point in the tree.
- if (this.treeCursor.startIndex === this.currentIndex) {
- this.pushOpenTag()
- this.descendLeft()
+ while (true) {
+ const {startIndex} = this.treeCursor
+ const scopeName = this.currentScopeName()
+ if (scopeName) {
+ this.openTags.push(this.idForScope(scopeName))
}
- break ascendingLoop
+ if (this.treeCursor.gotoFirstChild()) {
+ if ((this.closeTags.length || this.openTags.length) &&
+ this.treeCursor.startIndex > startIndex) {
+ this.treeCursor.gotoParent()
+ break
+ }
+
+ this.containingNodeTypes.push(this.treeCursor.nodeType)
+ this.containingNodeChildIndices.push(0)
+ this.containingNodeEndIndices.push(this.treeCursor.endIndex)
+ } else {
+ break
+ }
}
+ } else if (this.treeCursor.gotoParent()) {
+ this.atEnd = false
+ this.containingNodeTypes.pop()
+ this.containingNodeChildIndices.pop()
+ this.containingNodeEndIndices.pop()
+ } else {
+ this.done = true
+ break
+ }
+ } else {
+ this.atEnd = true
+ didMove = true
- this.currentChildIndex = last(this.containingNodeChildIndices)
- } while (this.treeCursor.gotoParent())
- /* eslint-disable no-labels */
+ const scopeName = this.currentScopeName()
+ if (scopeName) {
+ this.closeTags.push(this.idForScope(scopeName))
+ }
- // If the iterator is at the end of a node, advance to the node's next sibling. If
- // it has no next sibing, then the iterator has reached the end of the tree.
- } else if (!this.treeCursor.gotoNextSibling()) {
- this.currentPosition = {row: Infinity, column: Infinity}
- break
+ const endIndex = this.treeCursor.endIndex
+ let depth = this.containingNodeEndIndices.length
+ while (depth > 1 && this.containingNodeEndIndices[depth - 2] === endIndex) {
+ this.treeCursor.gotoParent()
+ this.containingNodeTypes.pop()
+ this.containingNodeChildIndices.pop()
+ this.containingNodeEndIndices.pop()
+ --depth
+ const scopeName = this.currentScopeName()
+ if (scopeName) this.closeTags.push(this.idForScope(scopeName))
+ }
}
- } while (this.closeTags.length === 0 && this.openTags.length === 0)
- return true
+ if (didMove && (this.closeTags.length || this.openTags.length)) break
+ }
}
getPosition () {
- return this.currentPosition
+ if (this.done) {
+ return Point.INFINITY
+ } else if (this.atEnd) {
+ return this.treeCursor.endPosition
+ } else {
+ return this.treeCursor.startPosition
+ }
+ }
+
+ getIndex () {
+ if (this.done) {
+ return Infinity
+ } else if (this.atEnd) {
+ return this.treeCursor.endIndex
+ } else {
+ return this.treeCursor.startIndex
+ }
}
getCloseScopeIds () {
@@ -532,34 +903,137 @@ class TreeSitterHighlightIterator {
// Private methods
- descendLeft () {
- while (this.treeCursor.gotoFirstChild()) {
- this.currentChildIndex = 0
- this.pushOpenTag()
- }
- }
-
currentScopeName () {
- return this.languageMode.grammar.scopeMap.get(
+ return this.languageLayer.grammar.scopeMap.get(
this.containingNodeTypes,
this.containingNodeChildIndices,
this.treeCursor.nodeIsNamed
)
}
- pushCloseTag () {
- const scopeName = this.currentScopeName()
- if (scopeName) this.closeTags.push(this.languageMode.grammar.idForScope(scopeName))
- this.containingNodeTypes.pop()
- this.containingNodeChildIndices.pop()
+ idForScope (scopeName) {
+ return this.languageLayer.languageMode.grammar.idForScope(scopeName)
+ }
+}
+
+class NullHighlightIterator {
+ seek () { return [] }
+ moveToSuccessor () {}
+ getIndex () { return Infinity }
+ getPosition () { return Point.INFINITY }
+ getOpenScopeIds () { return [] }
+ getCloseScopeIds () { return [] }
+}
+
+class NodeRangeSet {
+ constructor (previous, nodes) {
+ this.previous = previous
+ this.nodes = nodes
}
- pushOpenTag () {
- this.containingNodeTypes.push(this.treeCursor.nodeType)
- this.containingNodeChildIndices.push(this.currentChildIndex)
- const scopeName = this.currentScopeName()
- if (scopeName) this.openTags.push(this.languageMode.grammar.idForScope(scopeName))
+ getRanges () {
+ const previousRanges = this.previous && this.previous.getRanges()
+ const result = []
+
+ for (const node of this.nodes) {
+ let position = node.startPosition
+ let index = node.startIndex
+
+ for (const child of node.children) {
+ const nextPosition = child.startPosition
+ const nextIndex = child.startIndex
+ if (nextIndex > index) {
+ this._pushRange(previousRanges, result, {
+ startIndex: index,
+ endIndex: nextIndex,
+ startPosition: position,
+ endPosition: nextPosition
+ })
+ }
+ position = child.endPosition
+ index = child.endIndex
+ }
+
+ if (node.endIndex > index) {
+ this._pushRange(previousRanges, result, {
+ startIndex: index,
+ endIndex: node.endIndex,
+ startPosition: position,
+ endPosition: node.endPosition
+ })
+ }
+ }
+
+ return result
}
+
+ _pushRange (previousRanges, newRanges, newRange) {
+ if (!previousRanges) {
+ newRanges.push(newRange)
+ return
+ }
+
+ for (const previousRange of previousRanges) {
+ if (previousRange.endIndex <= newRange.startIndex) continue
+ if (previousRange.startIndex >= newRange.endIndex) break
+ newRanges.push({
+ startIndex: Math.max(previousRange.startIndex, newRange.startIndex),
+ endIndex: Math.min(previousRange.endIndex, newRange.endIndex),
+ startPosition: Point.max(previousRange.startPosition, newRange.startPosition),
+ endPosition: Point.min(previousRange.endPosition, newRange.endPosition)
+ })
+ }
+ }
+}
+
+function insertContainingTag (tag, index, tags, indices) {
+ const i = indices.findIndex(existingIndex => existingIndex > index)
+ if (i === -1) {
+ tags.push(tag)
+ indices.push(index)
+ } else {
+ tags.splice(i, 0, tag)
+ indices.splice(i, 0, index)
+ }
+}
+
+// Return true iff `mouse` is smaller than `house`. Only correct if
+// mouse and house overlap.
+//
+// * `mouse` {Range}
+// * `house` {Range}
+function rangeIsSmaller (mouse, house) {
+ if (!house) return true
+ const mvec = vecFromRange(mouse)
+ const hvec = vecFromRange(house)
+ return Point.min(mvec, hvec) === mvec
+}
+
+function vecFromRange ({start, end}) {
+ return end.translate(start.negate())
+}
+
+function rangeForNode (node) {
+ return new Range(node.startPosition, node.endPosition)
+}
+
+function nodeContainsIndices (node, start, end) {
+ if (node.startIndex < start) return node.endIndex >= end
+ if (node.startIndex === start) return node.endIndex > end
+ return false
+}
+
+function nodeIsSmaller (left, right) {
+ if (!left) return false
+ if (!right) return true
+ return left.endIndex - left.startIndex < right.endIndex - right.startIndex
+}
+
+function compareScopeDescriptorIterators (a, b) {
+ return (
+ a.node.startIndex - b.node.startIndex ||
+ a.rootStartIndex - b.rootStartIndex
+ )
}
function pointIsGreater (left, right) {
@@ -579,5 +1053,9 @@ function last (array) {
'decreaseNextIndentRegexForScopeDescriptor',
'regexForPattern'
].forEach(methodName => {
- module.exports.prototype[methodName] = TextMateLanguageMode.prototype[methodName]
+ TreeSitterLanguageMode.prototype[methodName] = TextMateLanguageMode.prototype[methodName]
})
+
+TreeSitterLanguageMode.LanguageLayer = LanguageLayer
+
+module.exports = TreeSitterLanguageMode