From 9204836d705105ce050bdb69691bb346449cf079 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Sat, 20 Apr 2013 19:25:03 -0600 Subject: [PATCH] Update grammars when grammars they include are added/removed If the Ruby on Rails grammar depends on HTML, but it isn't loaded, its syntax highlighting won't include HTMl tokens. If we later load HTML, we should update any buffer with the Rails grammar to reflect the change. This commit changes grammars to memoize their initial rule and repository. If an included grammar is added or removed, we clear the memoized rules and emit a 'grammar-updated' event. Any tokenized buffer that points to this grammar can then retokenize to reflect the newly available/unavailable included grammar. --- spec/app/text-mate-grammar-spec.coffee | 83 +++++++++++++++++--------- src/app/null-grammar.coffee | 2 + src/app/syntax.coffee | 6 ++ src/app/text-mate-grammar.coffee | 47 +++++++++++---- vendor/packages/php.tmbundle | 2 +- vendor/packages/sql.tmbundle | 1 + 6 files changed, 102 insertions(+), 39 deletions(-) create mode 160000 vendor/packages/sql.tmbundle diff --git a/spec/app/text-mate-grammar-spec.coffee b/spec/app/text-mate-grammar-spec.coffee index 5a0d66a4e..3dae2764a 100644 --- a/spec/app/text-mate-grammar-spec.coffee +++ b/spec/app/text-mate-grammar-spec.coffee @@ -210,36 +210,63 @@ describe "TextMateGrammar", -> expect(tokens[11]).toEqual value: ' damn.', scopes: ["source.ruby","comment.line.number-sign.ruby"] describe "when the pattern includes rules from another grammar", -> - it "parses tokens inside the begin/end patterns based on the included grammar's rules", -> - atom.activatePackage('html.tmbundle', sync: true) - atom.activatePackage('ruby-on-rails-tmbundle', sync: true) + describe "when a grammar matching the desired scope is available", -> + it "parses tokens inside the begin/end patterns based on the included grammar's rules", -> + atom.activatePackage('html.tmbundle', sync: true) + atom.activatePackage('ruby-on-rails-tmbundle', sync: true) - grammar = syntax.selectGrammar('foo.html.erb') - {tokens} = grammar.tokenizeLine("
<%= User.find(2).full_name %>
") + grammar = syntax.selectGrammar('foo.html.erb') + {tokens} = grammar.tokenizeLine("
<%= User.find(2).full_name %>
") - expect(tokens[0]).toEqual value: '<', scopes: ["text.html.ruby","meta.tag.block.any.html","punctuation.definition.tag.begin.html"] - expect(tokens[1]).toEqual value: 'div', scopes: ["text.html.ruby","meta.tag.block.any.html","entity.name.tag.block.any.html"] - expect(tokens[2]).toEqual value: ' ', scopes: ["text.html.ruby","meta.tag.block.any.html"] - expect(tokens[3]).toEqual value: 'class', scopes: ["text.html.ruby","meta.tag.block.any.html", "entity.other.attribute-name.html"] - expect(tokens[4]).toEqual value: '=', scopes: ["text.html.ruby","meta.tag.block.any.html"] - expect(tokens[5]).toEqual value: '\'', scopes: ["text.html.ruby","meta.tag.block.any.html","string.quoted.single.html","punctuation.definition.string.begin.html"] - expect(tokens[6]).toEqual value: 'name', scopes: ["text.html.ruby","meta.tag.block.any.html","string.quoted.single.html"] - expect(tokens[7]).toEqual value: '\'', scopes: ["text.html.ruby","meta.tag.block.any.html","string.quoted.single.html","punctuation.definition.string.end.html"] - expect(tokens[8]).toEqual value: '>', scopes: ["text.html.ruby","meta.tag.block.any.html","punctuation.definition.tag.end.html"] - expect(tokens[9]).toEqual value: '<%=', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","punctuation.section.embedded.ruby"] - expect(tokens[10]).toEqual value: ' ', scopes: ["text.html.ruby","source.ruby.rails.embedded.html"] - expect(tokens[11]).toEqual value: 'User', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","support.class.ruby"] - expect(tokens[12]).toEqual value: '.', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","punctuation.separator.method.ruby"] - expect(tokens[13]).toEqual value: 'find', scopes: ["text.html.ruby","source.ruby.rails.embedded.html"] - expect(tokens[14]).toEqual value: '(', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","punctuation.section.function.ruby"] - expect(tokens[15]).toEqual value: '2', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","constant.numeric.ruby"] - expect(tokens[16]).toEqual value: ')', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","punctuation.section.function.ruby"] - expect(tokens[17]).toEqual value: '.', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","punctuation.separator.method.ruby"] - expect(tokens[18]).toEqual value: 'full_name ', scopes: ["text.html.ruby","source.ruby.rails.embedded.html"] - expect(tokens[19]).toEqual value: '%>', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","punctuation.section.embedded.ruby"] - expect(tokens[20]).toEqual value: '', scopes: ["text.html.ruby","meta.tag.block.any.html","punctuation.definition.tag.end.html"] + expect(tokens[0]).toEqual value: '<', scopes: ["text.html.ruby","meta.tag.block.any.html","punctuation.definition.tag.begin.html"] + expect(tokens[1]).toEqual value: 'div', scopes: ["text.html.ruby","meta.tag.block.any.html","entity.name.tag.block.any.html"] + expect(tokens[2]).toEqual value: ' ', scopes: ["text.html.ruby","meta.tag.block.any.html"] + expect(tokens[3]).toEqual value: 'class', scopes: ["text.html.ruby","meta.tag.block.any.html", "entity.other.attribute-name.html"] + expect(tokens[4]).toEqual value: '=', scopes: ["text.html.ruby","meta.tag.block.any.html"] + expect(tokens[5]).toEqual value: '\'', scopes: ["text.html.ruby","meta.tag.block.any.html","string.quoted.single.html","punctuation.definition.string.begin.html"] + expect(tokens[6]).toEqual value: 'name', scopes: ["text.html.ruby","meta.tag.block.any.html","string.quoted.single.html"] + expect(tokens[7]).toEqual value: '\'', scopes: ["text.html.ruby","meta.tag.block.any.html","string.quoted.single.html","punctuation.definition.string.end.html"] + expect(tokens[8]).toEqual value: '>', scopes: ["text.html.ruby","meta.tag.block.any.html","punctuation.definition.tag.end.html"] + expect(tokens[9]).toEqual value: '<%=', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","punctuation.section.embedded.ruby"] + expect(tokens[10]).toEqual value: ' ', scopes: ["text.html.ruby","source.ruby.rails.embedded.html"] + expect(tokens[11]).toEqual value: 'User', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","support.class.ruby"] + expect(tokens[12]).toEqual value: '.', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","punctuation.separator.method.ruby"] + expect(tokens[13]).toEqual value: 'find', scopes: ["text.html.ruby","source.ruby.rails.embedded.html"] + expect(tokens[14]).toEqual value: '(', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","punctuation.section.function.ruby"] + expect(tokens[15]).toEqual value: '2', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","constant.numeric.ruby"] + expect(tokens[16]).toEqual value: ')', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","punctuation.section.function.ruby"] + expect(tokens[17]).toEqual value: '.', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","punctuation.separator.method.ruby"] + expect(tokens[18]).toEqual value: 'full_name ', scopes: ["text.html.ruby","source.ruby.rails.embedded.html"] + expect(tokens[19]).toEqual value: '%>', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","punctuation.section.embedded.ruby"] + expect(tokens[20]).toEqual value: '', scopes: ["text.html.ruby","meta.tag.block.any.html","punctuation.definition.tag.end.html"] + + describe "when a grammar matching the desired scope is unavailable", -> + it "updates the grammar if a matching grammar is added later", -> + atom.deactivatePackage('html.tmbundle') + atom.activatePackage('ruby-on-rails-tmbundle', sync: true) + + grammar = syntax.selectGrammar('foo.html.erb') + {tokens} = grammar.tokenizeLine("
<%= User.find(2).full_name %>
") + expect(tokens[0]).toEqual value: "
", scopes: ["text.html.ruby"] + expect(tokens[1]).toEqual value: '<%=', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","punctuation.section.embedded.ruby"] + expect(tokens[2]).toEqual value: ' ', scopes: ["text.html.ruby","source.ruby.rails.embedded.html"] + expect(tokens[3]).toEqual value: 'User', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","support.class.ruby"] + + atom.activatePackage('html.tmbundle', sync: true) + {tokens} = grammar.tokenizeLine("
<%= User.find(2).full_name %>
") + expect(tokens[0]).toEqual value: '<', scopes: ["text.html.ruby","meta.tag.block.any.html","punctuation.definition.tag.begin.html"] + expect(tokens[1]).toEqual value: 'div', scopes: ["text.html.ruby","meta.tag.block.any.html","entity.name.tag.block.any.html"] + expect(tokens[2]).toEqual value: ' ', scopes: ["text.html.ruby","meta.tag.block.any.html"] + expect(tokens[3]).toEqual value: 'class', scopes: ["text.html.ruby","meta.tag.block.any.html", "entity.other.attribute-name.html"] + expect(tokens[4]).toEqual value: '=', scopes: ["text.html.ruby","meta.tag.block.any.html"] + expect(tokens[5]).toEqual value: '\'', scopes: ["text.html.ruby","meta.tag.block.any.html","string.quoted.single.html","punctuation.definition.string.begin.html"] + expect(tokens[6]).toEqual value: 'name', scopes: ["text.html.ruby","meta.tag.block.any.html","string.quoted.single.html"] + expect(tokens[7]).toEqual value: '\'', scopes: ["text.html.ruby","meta.tag.block.any.html","string.quoted.single.html","punctuation.definition.string.end.html"] + expect(tokens[8]).toEqual value: '>', scopes: ["text.html.ruby","meta.tag.block.any.html","punctuation.definition.tag.end.html"] + expect(tokens[9]).toEqual value: '<%=', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","punctuation.section.embedded.ruby"] + expect(tokens[10]).toEqual value: ' ', scopes: ["text.html.ruby","source.ruby.rails.embedded.html"] it "can parse a grammar with newline characters in its regular expressions (regression)", -> grammar = new TextMateGrammar diff --git a/src/app/null-grammar.coffee b/src/app/null-grammar.coffee index 8b5e7b3e4..af9cd3ae8 100644 --- a/src/app/null-grammar.coffee +++ b/src/app/null-grammar.coffee @@ -12,3 +12,5 @@ class NullGrammar tokenizeLine: (line) -> { tokens: [new Token(value: line, scopes: ['null-grammar.text.plain'])] } + + grammarAddedOrRemoved: -> # no op diff --git a/src/app/syntax.coffee b/src/app/syntax.coffee index c7076fbd2..db48ad045 100644 --- a/src/app/syntax.coffee +++ b/src/app/syntax.coffee @@ -31,13 +31,19 @@ class Syntax { deserializer: @constructor.name, @grammarOverridesByPath } addGrammar: (grammar) -> + previousGrammars = new Array(@grammars...) @grammars.push(grammar) @grammarsByScopeName[grammar.scopeName] = grammar + @notifyOtherGrammars(previousGrammars, grammar.scopeName) @trigger 'grammar-added', grammar removeGrammar: (grammar) -> _.remove(@grammars, grammar) delete @grammarsByScopeName[grammar.scopeName] + @notifyOtherGrammars(@grammars, grammar.scopeName) + + notifyOtherGrammars: (grammars, scopeName) -> + grammar.grammarAddedOrRemoved(scopeName) for grammar in grammars setGrammarOverrideForPath: (path, scopeName) -> @grammarOverridesByPath[path] = scopeName diff --git a/src/app/text-mate-grammar.coffee b/src/app/text-mate-grammar.coffee index 039df3243..a38cb1074 100644 --- a/src/app/text-mate-grammar.coffee +++ b/src/app/text-mate-grammar.coffee @@ -4,6 +4,7 @@ plist = require 'plist' Token = require 'token' {OnigRegExp, OnigScanner} = require 'oniguruma' nodePath = require 'path' +EventEmitter = require 'event-emitter' pathSplitRegex = new RegExp("[#{nodePath.sep}.]") TextMateScopeSelector = require 'text-mate-scope-selector' @@ -27,23 +28,46 @@ class TextMateGrammar new TextMateGrammar(fsUtils.readObject(path)) name: null + rawPatterns: null + rawRepository: null fileTypes: null scopeName: null repository: null initialRule: null firstLineRegex: null + includedGrammarScopes: null maxTokensPerLine: 100 constructor: ({ @name, @fileTypes, @scopeName, injections, patterns, repository, @foldingStopMarker, firstLineMatch}) -> + @rawPatterns = patterns + @rawRepository = repository @injections = new Injections(this, injections) - @initialRule = new Rule(this, {@scopeName, patterns}) - @repository = {} @firstLineRegex = new OnigRegExp(firstLineMatch) if firstLineMatch @fileTypes ?= [] + @includedGrammarScopes = [] - for name, data of repository - data = {patterns: [data], tempName: name} if data.begin? or data.match? - @repository[name] = new Rule(this, data) + clearRules: -> + @initialRule = null + @repository = null + + getInitialRule: -> + @initialRule ?= new Rule(this, {@scopeName, patterns: @rawPatterns}) + + getRepository: -> + @repository ?= do => + repository = {} + for name, data of @rawRepository + data = {patterns: [data], tempName: name} if data.begin? or data.match? + repository[name] = new Rule(this, data) + repository + + addIncludedGrammarScope: (scope) -> + @includedGrammarScopes.push(scope) unless _.include(@includedGrammarScopes, scope) + + grammarAddedOrRemoved: (scopeName) => + return unless _.include(@includedGrammarScopes, scopeName) + @clearRules() + @trigger 'grammar-updated' getScore: (path, contents) -> contents = fsUtils.read(path) if not contents? and fsUtils.isFile(path) @@ -82,7 +106,7 @@ class TextMateGrammar pathSuffix = pathComponents[-fileTypeComponents.length..-1] _.isEqual(pathSuffix, fileTypeComponents) - tokenizeLine: (line, ruleStack=[@initialRule], firstLine=false) -> + tokenizeLine: (line, ruleStack=[@getInitialRule()], firstLine=false) -> originalRuleStack = ruleStack ruleStack = new Array(ruleStack...) # clone ruleStack tokens = [] @@ -173,6 +197,8 @@ class Injections scanners.push(scanner) scanners +_.extend TextMateGrammar.prototype, EventEmitter + class Rule grammar: null scopeName: null @@ -359,13 +385,14 @@ class Pattern ruleForInclude: (baseGrammar, name) -> if name[0] == "#" - @grammar.repository[name[1..]] + @grammar.getRepository()[name[1..]] else if name == "$self" - @grammar.initialRule + @grammar.getInitialRule() else if name == "$base" - baseGrammar.initialRule + baseGrammar.getInitialRule() else - syntax.grammarForScopeName(name)?.initialRule + @grammar.addIncludedGrammarScope(name) + syntax.grammarForScopeName(name)?.getInitialRule() getIncludedPatterns: (baseGrammar, included) -> if @include diff --git a/vendor/packages/php.tmbundle b/vendor/packages/php.tmbundle index 250026c5d..bb576b557 160000 --- a/vendor/packages/php.tmbundle +++ b/vendor/packages/php.tmbundle @@ -1 +1 @@ -Subproject commit 250026c5deef91f3d5a761be71fe09b740ec9ab5 +Subproject commit bb576b557518802c458d85bc2c0a7430fe90e1a0 diff --git a/vendor/packages/sql.tmbundle b/vendor/packages/sql.tmbundle new file mode 160000 index 000000000..1e047e07a --- /dev/null +++ b/vendor/packages/sql.tmbundle @@ -0,0 +1 @@ +Subproject commit 1e047e07ad45d67ee7e747fae17bee896eef638f