github/atom
1
1
Fork 0
mirror of https://github.com/atom/atom.git synced 2026-01-26 07:19:06 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:atom/atom into aw/apm-up

Browse Source
This commit is contained in:
Ash Wilson
2018-07-18 14:34:27 -04:00
parent c6a1ffedf0 643ffd6bb6
commit ab47adbd38
10 changed files with 1385 additions and 287 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
package.json
View File

@@ -157,9 +157,9 @@
"symbols-view": "https://www.atom.io/api/packages/symbols-view/versions/0.118.2/tarball",
"tabs": "https://www.atom.io/api/packages/tabs/versions/0.109.2/tarball",
"temp": "^0.8.3",
"text-buffer": "13.14.4",
"text-buffer": "13.14.5",
"timecop": "https://www.atom.io/api/packages/timecop/versions/0.36.2/tarball",
"tree-sitter": "0.12.12",
"tree-sitter": "0.12.20",
"tree-view": "https://www.atom.io/api/packages/tree-view/versions/0.222.0/tarball",
"typescript-simple": "1.0.0",
"underscore-plus": "^1.6.8",
@@ -203,7 +203,7 @@
"exception-reporting": "0.43.1",
"find-and-replace": "0.215.11",
"fuzzy-finder": "1.8.2",
"github": "0.17.3",
"github": "0.18.2",
"git-diff": "1.3.9",
"go-to-line": "0.33.0",
"grammar-selector": "0.50.1",
@@ -238,10 +238,10 @@
"language-gfm": "0.90.5",
"language-git": "0.19.1",
"language-go": "0.45.4",
"language-html": "0.50.1",
"language-html": "0.50.3",
"language-hyperlink": "0.16.3",
"language-java": "0.30.0",
"language-javascript": "0.128.8",
"language-javascript": "0.128.11",
"language-json": "0.19.2",
"language-less": "0.34.2",
"language-make": "0.22.3",

28
spec/grammar-registry-spec.js
View File

@@ -525,6 +525,34 @@ describe('GrammarRegistry', () => {
})
})
describe('.addInjectionPoint(languageId, {type, language, content})', () => {
const injectionPoint = {
type: 'some_node_type',
language() { return 'some_language_name' },
content(node) { return node }
}
beforeEach(() => {
atom.config.set('core.useTreeSitterParsers', true)
})
it('adds an injection point to the grammar with the given id', async () => {
await atom.packages.activatePackage('language-javascript')
atom.grammars.addInjectionPoint('javascript', injectionPoint)
const grammar = atom.grammars.grammarForId('javascript')
expect(grammar.injectionPoints).toContain(injectionPoint)
})
describe('when called before a grammar with the given id is loaded', () => {
it('adds the injection point once the grammar is loaded', async () => {
atom.grammars.addInjectionPoint('javascript', injectionPoint)
await atom.packages.activatePackage('language-javascript')
const grammar = atom.grammars.grammarForId('javascript')
expect(grammar.injectionPoints).toContain(injectionPoint)
})
})
})
describe('serialization', () => {
it('persists editors\' grammar overrides', async () => {
const buffer1 = new TextBuffer()

664
spec/tree-sitter-language-mode-spec.js
View File

@@ -11,6 +11,7 @@ const cGrammarPath = require.resolve('language-c/grammars/tree-sitter-c.cson')
const pythonGrammarPath = require.resolve('language-python/grammars/tree-sitter-python.cson')
const jsGrammarPath = require.resolve('language-javascript/grammars/tree-sitter-javascript.cson')
const htmlGrammarPath = require.resolve('language-html/grammars/tree-sitter-html.cson')
const ejsGrammarPath = require.resolve('language-html/grammars/tree-sitter-ejs.cson')
describe('TreeSitterLanguageMode', () => {
let editor, buffer
@@ -18,6 +19,7 @@ describe('TreeSitterLanguageMode', () => {
beforeEach(async () => {
editor = await atom.workspace.open('')
buffer = editor.getBuffer()
editor.displayLayer.reset({foldCharacter: '…'})
})
describe('highlighting', () => {
@@ -32,10 +34,11 @@ describe('TreeSitterLanguageMode', () => {
}
})
buffer.setText('aa.bbb = cc(d.eee());')
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText('aa.bbb = cc(d.eee());')
await languageMode.reparsePromise
await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [[
{text: 'aa.', scopes: ['source']},
@@ -61,10 +64,11 @@ describe('TreeSitterLanguageMode', () => {
}
})
buffer.setText('a = bb.ccc();')
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText('a = bb.ccc();')
await languageMode.reparsePromise
await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [[
{text: 'a', scopes: ['source', 'variable']},
@@ -87,17 +91,18 @@ describe('TreeSitterLanguageMode', () => {
}
})
buffer.setText('a\n .b();')
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText('a\n .b();')
await languageMode.reparsePromise
await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [
[
{text: 'a', scopes: ['variable']},
],
[
{text: ' ', scopes: ['whitespace']},
{text: ' ', scopes: ['leading-whitespace']},
{text: '.', scopes: []},
{text: 'b', scopes: ['function']},
{text: '();', scopes: []}
@@ -114,12 +119,12 @@ describe('TreeSitterLanguageMode', () => {
}
})
buffer.setText('int main() {\n int a\n int b;\n}');
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText('int main() {\n int a\n int b;\n}');
await languageMode.reparsePromise
await nextHighlightingUpdate(languageMode)
editor.screenLineForScreenRow(0)
expect(
languageMode.tree.rootNode.descendantForPosition(Point(1, 2), Point(1, 6)).toString()
).toBe('(declaration (primitive_type) (identifier) (MISSING))')
@@ -132,13 +137,13 @@ describe('TreeSitterLanguageMode', () => {
{text: '() {', scopes: []}
],
[
{text: ' ', scopes: ['whitespace']},
{text: ' ', scopes: ['leading-whitespace']},
{text: 'int', scopes: ['type']},
{text: ' ', scopes: []},
{text: 'a', scopes: ['variable']}
],
[
{text: ' ', scopes: ['whitespace']},
{text: ' ', scopes: ['leading-whitespace']},
{text: 'int', scopes: ['type']},
{text: ' ', scopes: []},
{text: 'b', scopes: ['variable']},
@@ -159,10 +164,11 @@ describe('TreeSitterLanguageMode', () => {
}
})
buffer.setText('a(\nb,\nc\n')
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText('a(\nb,\nc\n')
await languageMode.reparsePromise
await nextHighlightingUpdate(languageMode)
// missing closing paren
expectTokensToEqual(editor, [
@@ -173,7 +179,7 @@ describe('TreeSitterLanguageMode', () => {
])
buffer.append(')')
await languageMode.reparsePromise
await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [
[
{text: 'a', scopes: ['function']},
@@ -195,15 +201,16 @@ describe('TreeSitterLanguageMode', () => {
}
})
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText([
'// abc',
'',
'a("b").c'
].join('\r\n'))
await languageMode.reparsePromise
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [
[{text: '// abc', scopes: ['comment']}],
[{text: '', scopes: []}],
@@ -216,12 +223,12 @@ describe('TreeSitterLanguageMode', () => {
])
buffer.insert([2, 0], ' ')
await languageMode.reparsePromise
await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [
[{text: '// abc', scopes: ['comment']}],
[{text: '', scopes: []}],
[
{text: ' ', scopes: ['whitespace']},
{text: ' ', scopes: ['leading-whitespace']},
{text: 'a(', scopes: []},
{text: '"b"', scopes: ['string']},
{text: ').', scopes: []},
@@ -230,6 +237,82 @@ describe('TreeSitterLanguageMode', () => {
])
})
it('handles multi-line nodes with children on different lines (regression)', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
scopes: {
'template_string': 'string',
'"${"': 'interpolation',
'"}"': 'interpolation'
}
});
buffer.setText('`\na${1}\nb${2}\n`;')
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [
[
{text: '`', scopes: ['string']}
], [
{text: 'a', scopes: ['string']},
{text: '${', scopes: ['string', 'interpolation']},
{text: '1', scopes: ['string']},
{text: '}', scopes: ['string', 'interpolation']}
], [
{text: 'b', scopes: ['string']},
{text: '${', scopes: ['string', 'interpolation']},
{text: '2', scopes: ['string']},
{text: '}', scopes: ['string', 'interpolation']}
],
[
{text: '`', scopes: ['string']},
{text: ';', scopes: []}
]
])
})
it('handles folds inside of highlighted tokens', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
scopes: {
'comment': 'comment',
'call_expression > identifier': 'function',
}
})
buffer.setText(dedent `
/*
* Hello
*/
hello();
`)
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
editor.foldBufferRange([[0, 2], [2, 0]])
expectTokensToEqual(editor, [
[
{text: '/*', scopes: ['comment']},
{text: '…', scopes: ['fold-marker']},
{text: ' */', scopes: ['comment']}
],
[
{text: '', scopes: []}
],
[
{text: 'hello', scopes: ['function']},
{text: '();', scopes: []},
]
])
})
describe('when the buffer changes during a parse', () => {
it('immediately parses again when the current parse completes', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
@@ -240,11 +323,14 @@ describe('TreeSitterLanguageMode', () => {
'new_expression > call_expression > identifier': 'constructor'
}
})
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText('abc;');
await languageMode.reparsePromise
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
await new Promise(process.nextTick)
expectTokensToEqual(editor, [
[
{text: 'abc', scopes: ['variable']},
@@ -269,8 +355,7 @@ describe('TreeSitterLanguageMode', () => {
],
])
await languageMode.reparsePromise
expect(languageMode.reparsePromise).not.toBeNull()
await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [
[
{text: 'new ', scopes: []},
@@ -279,8 +364,7 @@ describe('TreeSitterLanguageMode', () => {
],
])
await languageMode.reparsePromise
expect(languageMode.reparsePromise).toBeNull()
await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [
[
{text: 'new ', scopes: []},
@@ -290,13 +374,265 @@ describe('TreeSitterLanguageMode', () => {
])
})
})
describe('injectionPoints and injectionPatterns', () => {
let jsGrammar, htmlGrammar
beforeEach(() => {
jsGrammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
id: 'javascript',
parser: 'tree-sitter-javascript',
scopes: {
'property_identifier': 'property',
'call_expression > identifier': 'function',
'template_string': 'string',
'template_substitution > "${"': 'interpolation',
'template_substitution > "}"': 'interpolation'
},
injectionRegExp: 'javascript',
injectionPoints: [HTML_TEMPLATE_LITERAL_INJECTION_POINT]
})
htmlGrammar = new TreeSitterGrammar(atom.grammars, htmlGrammarPath, {
id: 'html',
parser: 'tree-sitter-html',
scopes: {
fragment: 'html',
tag_name: 'tag',
attribute_name: 'attr'
},
injectionRegExp: 'html',
injectionPoints: [SCRIPT_TAG_INJECTION_POINT]
})
})
it('highlights code inside of injection points', async () => {
atom.grammars.addGrammar(jsGrammar)
atom.grammars.addGrammar(htmlGrammar)
buffer.setText('node.innerHTML = html `\na ${b}<img src="d">\n`;')
const languageMode = new TreeSitterLanguageMode({buffer, grammar: jsGrammar, grammars: atom.grammars})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [
[
{text: 'node.', scopes: []},
{text: 'innerHTML', scopes: ['property']},
{text: ' = ', scopes: []},
{text: 'html', scopes: ['function']},
{text: ' ', scopes: []},
{text: '`', scopes: ['string']},
{text: '', scopes: ['string', 'html']}
], [
{text: 'a ', scopes: ['string', 'html']},
{text: '${', scopes: ['string', 'html', 'interpolation']},
{text: 'b', scopes: ['string', 'html']},
{text: '}', scopes: ['string', 'html', 'interpolation']},
{text: '<', scopes: ['string', 'html']},
{text: 'img', scopes: ['string', 'html', 'tag']},
{text: ' ', scopes: ['string', 'html']},
{text: 'src', scopes: ['string', 'html', 'attr']},
{text: '="d">', scopes: ['string', 'html']}
], [
{text: '`', scopes: ['string']},
{text: ';', scopes: []},
],
])
const range = buffer.findSync('html')
buffer.setTextInRange(range, 'xml')
await nextHighlightingUpdate(languageMode)
await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [
[
{text: 'node.', scopes: []},
{text: 'innerHTML', scopes: ['property']},
{text: ' = ', scopes: []},
{text: 'xml', scopes: ['function']},
{text: ' ', scopes: []},
{text: '`', scopes: ['string']}
], [
{text: 'a ', scopes: ['string']},
{text: '${', scopes: ['string', 'interpolation']},
{text: 'b', scopes: ['string']},
{text: '}', scopes: ['string', 'interpolation']},
{text: '<img src="d">', scopes: ['string']},
], [
{text: '`', scopes: ['string']},
{text: ';', scopes: []},
],
])
})
it('highlights the content after injections', async () => {
atom.grammars.addGrammar(jsGrammar)
atom.grammars.addGrammar(htmlGrammar)
buffer.setText('<script>\nhello();\n</script>\n<div>\n</div>')
const languageMode = new TreeSitterLanguageMode({buffer, grammar: htmlGrammar, grammars: atom.grammars})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [
[
{text: '<', scopes: ['html']},
{text: 'script', scopes: ['html', 'tag']},
{text: '>', scopes: ['html']},
],
[
{text: 'hello', scopes: ['html', 'function']},
{text: '();', scopes: ['html']},
],
[
{text: '</', scopes: ['html']},
{text: 'script', scopes: ['html', 'tag']},
{text: '>', scopes: ['html']},
],
[
{text: '<', scopes: ['html']},
{text: 'div', scopes: ['html', 'tag']},
{text: '>', scopes: ['html']},
],
[
{text: '</', scopes: ['html']},
{text: 'div', scopes: ['html', 'tag']},
{text: '>', scopes: ['html']},
]
])
})
it('updates buffers highlighting when a grammar with injectionRegExp is added', async () => {
atom.grammars.addGrammar(jsGrammar)
buffer.setText('node.innerHTML = html `\na ${b}<img src="d">\n`;')
const languageMode = new TreeSitterLanguageMode({buffer, grammar: jsGrammar, grammars: atom.grammars})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [
[
{text: 'node.', scopes: []},
{text: 'innerHTML', scopes: ['property']},
{text: ' = ', scopes: []},
{text: 'html', scopes: ['function']},
{text: ' ', scopes: []},
{text: '`', scopes: ['string']}
], [
{text: 'a ', scopes: ['string']},
{text: '${', scopes: ['string', 'interpolation']},
{text: 'b', scopes: ['string']},
{text: '}', scopes: ['string', 'interpolation']},
{text: '<img src="d">', scopes: ['string']},
], [
{text: '`', scopes: ['string']},
{text: ';', scopes: []},
],
])
atom.grammars.addGrammar(htmlGrammar)
await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [
[
{text: 'node.', scopes: []},
{text: 'innerHTML', scopes: ['property']},
{text: ' = ', scopes: []},
{text: 'html', scopes: ['function']},
{text: ' ', scopes: []},
{text: '`', scopes: ['string']},
{text: '', scopes: ['string', 'html']}
], [
{text: 'a ', scopes: ['string', 'html']},
{text: '${', scopes: ['string', 'html', 'interpolation']},
{text: 'b', scopes: ['string', 'html']},
{text: '}', scopes: ['string', 'html', 'interpolation']},
{text: '<', scopes: ['string', 'html']},
{text: 'img', scopes: ['string', 'html', 'tag']},
{text: ' ', scopes: ['string', 'html']},
{text: 'src', scopes: ['string', 'html', 'attr']},
{text: '="d">', scopes: ['string', 'html']}
], [
{text: '`', scopes: ['string']},
{text: ';', scopes: []},
],
])
})
it('handles injections that intersect', async () => {
const ejsGrammar = new TreeSitterGrammar(atom.grammars, ejsGrammarPath, {
id: 'ejs',
parser: 'tree-sitter-embedded-template',
scopes: {
'"<%="': 'directive',
'"%>"': 'directive',
},
injectionPoints: [
{
type: 'template',
language (node) { return 'javascript' },
content (node) { return node.descendantsOfType('code') }
},
{
type: 'template',
language (node) { return 'html' },
content (node) { return node.descendantsOfType('content') }
}
]
})
atom.grammars.addGrammar(jsGrammar)
atom.grammars.addGrammar(htmlGrammar)
buffer.setText('<body>\n<script>\nb(<%= c.d %>)\n</script>\n</body>')
const languageMode = new TreeSitterLanguageMode({buffer, grammar: ejsGrammar, grammars: atom.grammars})
buffer.setLanguageMode(languageMode)
// 4 parses: EJS, HTML, template JS, script tag JS
await nextHighlightingUpdate(languageMode)
await nextHighlightingUpdate(languageMode)
await nextHighlightingUpdate(languageMode)
await nextHighlightingUpdate(languageMode)
expectTokensToEqual(editor, [
[
{text: '<', scopes: ['html']},
{text: 'body', scopes: ['html', 'tag']},
{text: '>', scopes: ['html']}
],
[
{text: '<', scopes: ['html']},
{text: 'script', scopes: ['html', 'tag']},
{text: '>', scopes: ['html']}
],
[
{text: 'b', scopes: ['html', 'function']},
{text: '(', scopes: ['html']},
{text: '<%=', scopes: ['html', 'directive']},
{text: ' c.', scopes: ['html']},
{text: 'd', scopes: ['html', 'property']},
{text: ' ', scopes: ['html']},
{text: '%>', scopes: ['html', 'directive']},
{text: ')', scopes: ['html']},
],
[
{text: '</', scopes: ['html']},
{text: 'script', scopes: ['html', 'tag']},
{text: '>', scopes: ['html']}
],
[
{text: '</', scopes: ['html']},
{text: 'body', scopes: ['html', 'tag']},
{text: '>', scopes: ['html']}
],
])
})
})
})
describe('folding', () => {
beforeEach(() => {
editor.displayLayer.reset({foldCharacter: '…'})
})
it('can fold nodes that start and end with specified tokens', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
@@ -312,8 +648,6 @@ describe('TreeSitterLanguageMode', () => {
]
})
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText(dedent `
module.exports =
class A {
@@ -324,9 +658,10 @@ describe('TreeSitterLanguageMode', () => {
}
}
`)
await languageMode.reparsePromise
editor.screenLineForScreenRow(0)
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
expect(editor.isFoldableAtBufferRow(0)).toBe(false)
expect(editor.isFoldableAtBufferRow(1)).toBe(true)
@@ -375,8 +710,6 @@ describe('TreeSitterLanguageMode', () => {
]
})
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText(dedent `
const element1 = <Element
className='submit'
@@ -387,9 +720,10 @@ describe('TreeSitterLanguageMode', () => {
<span>world</span>
</Element>
`)
await languageMode.reparsePromise
editor.screenLineForScreenRow(0)
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
expect(editor.isFoldableAtBufferRow(0)).toBe(true)
expect(editor.isFoldableAtBufferRow(1)).toBe(false)
@@ -427,8 +761,6 @@ describe('TreeSitterLanguageMode', () => {
]
})
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText(dedent `
/**
* Important
@@ -437,9 +769,10 @@ describe('TreeSitterLanguageMode', () => {
Also important
*/
`)
await languageMode.reparsePromise
editor.screenLineForScreenRow(0)
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
expect(editor.isFoldableAtBufferRow(0)).toBe(true)
expect(editor.isFoldableAtBufferRow(1)).toBe(false)
@@ -488,8 +821,6 @@ describe('TreeSitterLanguageMode', () => {
]
})
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText(dedent `
#ifndef FOO_H_
#define FOO_H_
@@ -513,9 +844,10 @@ describe('TreeSitterLanguageMode', () => {
#endif
`)
await languageMode.reparsePromise
editor.screenLineForScreenRow(0)
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
editor.foldBufferRow(3)
expect(getDisplayText(editor)).toBe(dedent `
@@ -588,8 +920,6 @@ describe('TreeSitterLanguageMode', () => {
]
})
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText(dedent `
<head>
<meta name='key-1', content='value-1'>
@@ -597,7 +927,9 @@ describe('TreeSitterLanguageMode', () => {
</head>
`)
await languageMode.reparsePromise
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
// Void elements have only one child
expect(editor.isFoldableAtBufferRow(1)).toBe(false)
@@ -611,7 +943,7 @@ describe('TreeSitterLanguageMode', () => {
})
describe('when folding a node that ends with a line break', () => {
it('ends the fold at the end of the previous line', () => {
it('ends the fold at the end of the previous line', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, pythonGrammarPath, {
parser: 'tree-sitter-python',
folds: [
@@ -631,9 +963,9 @@ describe('TreeSitterLanguageMode', () => {
print 'c'
print 'd'
`)
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
editor.screenLineForScreenRow(0)
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
await nextHighlightingUpdate(buffer.getLanguageMode())
editor.foldBufferRow(0)
expect(getDisplayText(editor)).toBe(dedent `
@@ -645,19 +977,95 @@ describe('TreeSitterLanguageMode', () => {
`)
})
})
it('folds code in injected languages', async () => {
const htmlGrammar = new TreeSitterGrammar(atom.grammars, htmlGrammarPath, {
id: 'html',
parser: 'tree-sitter-html',
scopes: {},
folds: [{
type: ['element', 'raw_element'],
start: {index: 0},
end: {index: -1}
}],
injectionRegExp: 'html'
})
const jsGrammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
id: 'javascript',
parser: 'tree-sitter-javascript',
scopes: {},
folds: [{
type: ['template_string'],
start: {index: 0},
end: {index: -1},
},
{
start: {index: 0, type: '('},
end: {index: -1, type: ')'}
}],
injectionRegExp: 'javascript',
injectionPoints: [HTML_TEMPLATE_LITERAL_INJECTION_POINT]
})
atom.grammars.addGrammar(htmlGrammar)
buffer.setText(
`a = html \`
<div>
c\${def(
1,
2,
3,
)}e\${f}g
</div>
\`
`
)
const languageMode = new TreeSitterLanguageMode({buffer, grammar: jsGrammar, grammars: atom.grammars})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
await nextHighlightingUpdate(languageMode)
editor.foldBufferRow(2)
expect(getDisplayText(editor)).toBe(
`a = html \`
<div>
c\${def(…)}e\${f}g
</div>
\`
`
)
editor.foldBufferRow(1)
expect(getDisplayText(editor)).toBe(
`a = html \`
<div>…
</div>
\`
`
)
editor.foldBufferRow(0)
expect(getDisplayText(editor)).toBe(
`a = html \`…\`
`
)
})
})
describe('.scopeDescriptorForPosition', () => {
it('returns a scope descriptor representing the given position in the syntax tree', () => {
it('returns a scope descriptor representing the given position in the syntax tree', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
id: 'javascript',
parser: 'tree-sitter-javascript'
})
buffer.setText('foo({bar: baz});')
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
editor.screenLineForScreenRow(0)
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
await nextHighlightingUpdate(buffer.getLanguageMode())
expect(editor.scopeDescriptorForBufferPosition([0, 6]).getScopesArray()).toEqual([
'javascript',
'program',
@@ -669,10 +1077,65 @@ describe('TreeSitterLanguageMode', () => {
'property_identifier'
])
})
it('includes nodes in injected syntax trees', async () => {
const jsGrammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
id: 'javascript',
parser: 'tree-sitter-javascript',
scopes: {},
injectionRegExp: 'javascript',
injectionPoints: [HTML_TEMPLATE_LITERAL_INJECTION_POINT]
})
const htmlGrammar = new TreeSitterGrammar(atom.grammars, htmlGrammarPath, {
id: 'html',
parser: 'tree-sitter-html',
scopes: {},
injectionRegExp: 'html',
injectionPoints: [SCRIPT_TAG_INJECTION_POINT]
})
atom.grammars.addGrammar(jsGrammar)
atom.grammars.addGrammar(htmlGrammar)
buffer.setText(`
<div>
<script>
html \`
<span>\${person.name}</span>
\`
</script>
</div>
`)
const languageMode = new TreeSitterLanguageMode({buffer, grammar: htmlGrammar, grammars: atom.grammars})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
await nextHighlightingUpdate(languageMode)
await nextHighlightingUpdate(languageMode)
const position = buffer.findSync('name').start
expect(languageMode.scopeDescriptorForPosition(position).getScopesArray()).toEqual([
'html',
'fragment',
'element',
'raw_element',
'raw_text',
'program',
'expression_statement',
'call_expression',
'template_string',
'fragment',
'element',
'template_substitution',
'member_expression',
'property_identifier'
])
})
})
describe('TextEditor.selectLargerSyntaxNode and .selectSmallerSyntaxNode', () => {
it('expands and contract the selection based on the syntax tree', () => {
it('expands and contracts the selection based on the syntax tree', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
scopes: {'program': 'source'}
@@ -684,9 +1147,9 @@ describe('TreeSitterLanguageMode', () => {
g()
}
`)
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
editor.screenLineForScreenRow(0)
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
await nextHighlightingUpdate(buffer.getLanguageMode())
editor.setCursorBufferPosition([1, 3])
editor.selectLargerSyntaxNode()
@@ -711,9 +1174,72 @@ describe('TreeSitterLanguageMode', () => {
editor.selectSmallerSyntaxNode()
expect(editor.getSelectedBufferRange()).toEqual([[1, 3], [1, 3]])
})
it('handles injected languages', async () => {
const jsGrammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
id: 'javascript',
parser: 'tree-sitter-javascript',
scopes: {
'property_identifier': 'property',
'call_expression > identifier': 'function',
'template_string': 'string',
'template_substitution > "${"': 'interpolation',
'template_substitution > "}"': 'interpolation'
},
injectionRegExp: 'javascript',
injectionPoints: [HTML_TEMPLATE_LITERAL_INJECTION_POINT]
})
const htmlGrammar = new TreeSitterGrammar(atom.grammars, htmlGrammarPath, {
id: 'html',
parser: 'tree-sitter-html',
scopes: {
fragment: 'html',
tag_name: 'tag',
attribute_name: 'attr'
},
injectionRegExp: 'html'
})
atom.grammars.addGrammar(htmlGrammar)
buffer.setText('a = html ` <b>c${def()}e${f}g</b> `')
const languageMode = new TreeSitterLanguageMode({buffer, grammar: jsGrammar, grammars: atom.grammars})
buffer.setLanguageMode(languageMode)
await nextHighlightingUpdate(languageMode)
await nextHighlightingUpdate(languageMode)
editor.setCursorBufferPosition({row: 0, column: buffer.getText().indexOf('ef()')})
editor.selectLargerSyntaxNode()
expect(editor.getSelectedText()).toBe('def')
editor.selectLargerSyntaxNode()
expect(editor.getSelectedText()).toBe('def()')
editor.selectLargerSyntaxNode()
expect(editor.getSelectedText()).toBe('${def()}')
editor.selectLargerSyntaxNode()
expect(editor.getSelectedText()).toBe('c${def()}e${f}g')
editor.selectLargerSyntaxNode()
expect(editor.getSelectedText()).toBe('<b>c${def()}e${f}g</b>')
editor.selectLargerSyntaxNode()
expect(editor.getSelectedText()).toBe(' <b>c${def()}e${f}g</b> ')
editor.selectLargerSyntaxNode()
expect(editor.getSelectedText()).toBe('` <b>c${def()}e${f}g</b> `')
editor.selectLargerSyntaxNode()
expect(editor.getSelectedText()).toBe('html ` <b>c${def()}e${f}g</b> `')
})
})
})
function nextHighlightingUpdate (languageMode) {
return new Promise(resolve => {
const subscription = languageMode.onDidChangeHighlighting(() => {
subscription.dispose()
resolve()
})
})
}
function getDisplayText (editor) {
return editor.displayLayer.getText()
}
@@ -740,7 +1266,7 @@ function expectTokensToEqual (editor, expectedTokenLines) {
text,
scopes: scopes.map(scope => scope
.split(' ')
.map(className => className.slice('syntax--'.length))
.map(className => className.replace('syntax--', ''))
.join(' '))
}))
}
@@ -760,3 +1286,21 @@ function expectTokensToEqual (editor, expectedTokenLines) {
// due to subsequent edits can be tested.
editor.displayLayer.getScreenLines(0, Infinity)
}
const HTML_TEMPLATE_LITERAL_INJECTION_POINT = {
type: 'call_expression',
language (node) {
if (node.lastChild.type === 'template_string' && node.firstChild.type === 'identifier') {
return node.firstChild.text
}
},
content (node) {
return node.lastChild
}
}
const SCRIPT_TAG_INJECTION_POINT = {
type: 'raw_element',
language () { return 'javascript' },
content (node) { return node.child(1) }
}

55
src/grammar-registry.js
View File

@@ -169,7 +169,7 @@ class GrammarRegistry {
languageModeForGrammarAndBuffer (grammar, buffer) {
if (grammar instanceof TreeSitterGrammar) {
return new TreeSitterLanguageMode({grammar, buffer, config: this.config})
return new TreeSitterLanguageMode({grammar, buffer, config: this.config, grammars: this})
} else {
return new TextMateLanguageMode({grammar, buffer, config: this.config})
}
@@ -291,8 +291,9 @@ class GrammarRegistry {
forEachGrammar (callback) {
this.textmateRegistry.grammars.forEach(callback)
for (let grammarId in this.treeSitterGrammarsById) {
callback(this.treeSitterGrammarsById[grammarId])
for (const grammarId in this.treeSitterGrammarsById) {
const grammar = this.treeSitterGrammarsById[grammarId]
if (grammar.id) callback(grammar)
}
}
@@ -347,26 +348,23 @@ class GrammarRegistry {
this.grammarScoresByBuffer.forEach((score, buffer) => {
const languageMode = buffer.getLanguageMode()
if (grammar.injectionSelector) {
if (languageMode.hasTokenForSelector(grammar.injectionSelector)) {
languageMode.retokenizeLines()
}
return
}
const languageOverride = this.languageOverridesByBufferId.get(buffer.id)
if ((grammar.id === buffer.getLanguageMode().getLanguageId() ||
grammar.id === languageOverride)) {
buffer.setLanguageMode(this.languageModeForGrammarAndBuffer(grammar, buffer))
return
} else if (!languageOverride) {
const score = this.getGrammarScore(grammar, buffer.getPath(), getGrammarSelectionContent(buffer))
const currentScore = this.grammarScoresByBuffer.get(buffer)
if (currentScore == null || score > currentScore) {
buffer.setLanguageMode(this.languageModeForGrammarAndBuffer(grammar, buffer))
this.grammarScoresByBuffer.set(buffer, score)
return
}
}
languageMode.updateForInjection(grammar)
})
}
@@ -391,6 +389,32 @@ class GrammarRegistry {
return this.textmateRegistry.onDidUpdateGrammar(callback)
}
// Experimental: Specify a type of syntax node that may embed other languages.
//
// * `grammarId` The {String} id of the parent language
// * `injectionPoint` An {Object} with the following keys:
// * `type` The {String} type of syntax node that may embed other languages
// * `language` A {Function} that is called with syntax nodes of the specified `type` and
// returns a {String} that will be tested against other grammars' `injectionRegExp` in
// order to determine what language should be embedded.
// * `content` A {Function} that is called with syntax nodes of the specified `type` and
// returns another syntax node or array of syntax nodes that contain the embedded source code.
addInjectionPoint (grammarId, injectionPoint) {
const grammar = this.treeSitterGrammarsById[grammarId]
if (grammar) {
grammar.injectionPoints.push(injectionPoint)
} else {
this.treeSitterGrammarsById[grammarId] = {
injectionPoints: [injectionPoint]
}
}
return new Disposable(() => {
const grammar = this.treeSitterGrammarsById[grammarId]
const index = grammar.injectionPoints.indexOf(injectionPoint)
if (index !== -1) grammar.injectionPoints.splice(index, 1)
})
}
get nullGrammar () {
return this.textmateRegistry.nullGrammar
}
@@ -409,12 +433,14 @@ class GrammarRegistry {
addGrammar (grammar) {
if (grammar instanceof TreeSitterGrammar) {
const existingParams = this.treeSitterGrammarsById[grammar.id] || {}
this.treeSitterGrammarsById[grammar.id] = grammar
if (grammar.legacyScopeName) {
this.config.setLegacyScopeAliasForNewScope(grammar.id, grammar.legacyScopeName)
this.textMateScopeNamesByTreeSitterLanguageId.set(grammar.id, grammar.legacyScopeName)
this.treeSitterLanguageIdsByTextMateScopeName.set(grammar.legacyScopeName, grammar.id)
}
if (existingParams.injectionPoints) grammar.injectionPoints.push(...existingParams.injectionPoints)
this.grammarAddedOrUpdated(grammar)
return new Disposable(() => this.removeGrammar(grammar))
} else {
@@ -515,6 +541,15 @@ class GrammarRegistry {
return this.textmateRegistry.scopeForId(id)
}
treeSitterGrammarForLanguageString (languageString) {
for (const id in this.treeSitterGrammarsById) {
const grammar = this.treeSitterGrammarsById[id]
if (grammar.injectionRegExp && grammar.injectionRegExp.test(languageString)) {
return grammar
}
}
}
normalizeLanguageId (languageId) {
if (this.config.get('core.useTreeSitterParsers')) {
return this.treeSitterLanguageIdsByTextMateScopeName.get(languageId) || languageId

2
src/initialize-application-window.coffee
View File

@@ -36,6 +36,8 @@ if global.isGeneratingSnapshot
require('image-view')
require('incompatible-packages')
require('keybinding-resolver')
require('language-html')
require('language-javascript')
require('line-ending-selector')
require('link')
require('markdown-preview')

3
src/project.js
View File

@@ -685,9 +685,6 @@ class Project extends Model {
}
this.grammarRegistry.autoAssignLanguageMode(buffer)
if (buffer.languageMode.initialize) {
await buffer.languageMode.initialize()
}
this.addBuffer(buffer)
return buffer

9
src/test.ejs Normal file
View File

@@ -0,0 +1,9 @@
<html>
<% if something() { %>
<div class=foo>
<%= html `ok how about <span>this</span>` %>
</div>
<% } %>
</html>

9
src/text-mate-language-mode.js
View File

@@ -235,15 +235,18 @@ class TextMateLanguageMode {
return this.buffer.getTextInRange([[0, 0], [10, 0]])
}
hasTokenForSelector (selector) {
updateForInjection (grammar) {
if (!grammar.injectionSelector) return
for (const tokenizedLine of this.tokenizedLines) {
if (tokenizedLine) {
for (let token of tokenizedLine.tokens) {
if (selector.matches(token.scopes)) return true
if (grammar.injectionSelector.matches(token.scopes)) {
this.retokenizeLines()
return
}
}
}
}
return false
}
retokenizeLines () {

2
src/tree-sitter-grammar.js
View File

@@ -10,6 +10,7 @@ class TreeSitterGrammar {
this.name = params.name
this.legacyScopeName = params.legacyScopeName
if (params.contentRegExp) this.contentRegExp = new RegExp(params.contentRegExp)
if (params.injectionRegExp) this.injectionRegExp = new RegExp(params.injectionRegExp)
this.folds = params.folds || []
@@ -28,6 +29,7 @@ class TreeSitterGrammar {
this.scopeMap = new SyntaxScopeMap(scopeSelectors)
this.fileTypes = params.fileTypes
this.injectionPoints = params.injectionPoints || []
// TODO - When we upgrade to a new enough version of node, use `require.resolve`
// with the new `paths` option instead of this private API.

890
src/tree-sitter-language-mode.js
View File

File diff suppressed because it is too large Load Diff