Update grammars when grammars they include are added/removed

If the Ruby on Rails grammar depends on HTML, but it isn't loaded, its
syntax highlighting won't include HTMl tokens. If we later load HTML,
we should update any buffer with the Rails grammar to reflect the
change. This commit changes grammars to memoize their initial rule and
repository. If an included grammar is added or removed, we clear the
memoized rules and emit a 'grammar-updated' event. Any tokenized
buffer that points to this grammar can then retokenize to reflect the
newly available/unavailable included grammar.

spec/app/text-mate-grammar-spec.coffee

@@ -210,36 +210,63 @@ describe "TextMateGrammar", ->
           expect(tokens[11]).toEqual value: ' damn.', scopes: ["source.ruby","comment.line.number-sign.ruby"]
 
       describe "when the pattern includes rules from another grammar", ->
-        it "parses tokens inside the begin/end patterns based on the included grammar's rules", ->
-          atom.activatePackage('html.tmbundle', sync: true)
-          atom.activatePackage('ruby-on-rails-tmbundle', sync: true)
+        describe "when a grammar matching the desired scope is available", ->
+          it "parses tokens inside the begin/end patterns based on the included grammar's rules", ->
+            atom.activatePackage('html.tmbundle', sync: true)
+            atom.activatePackage('ruby-on-rails-tmbundle', sync: true)
 
-          grammar = syntax.selectGrammar('foo.html.erb')
-          {tokens} = grammar.tokenizeLine("<div class='name'><%= User.find(2).full_name %></div>")
+            grammar = syntax.selectGrammar('foo.html.erb')
+            {tokens} = grammar.tokenizeLine("<div class='name'><%= User.find(2).full_name %></div>")
 
-          expect(tokens[0]).toEqual value: '<', scopes: ["text.html.ruby","meta.tag.block.any.html","punctuation.definition.tag.begin.html"]
-          expect(tokens[1]).toEqual value: 'div', scopes: ["text.html.ruby","meta.tag.block.any.html","entity.name.tag.block.any.html"]
-          expect(tokens[2]).toEqual value: ' ', scopes: ["text.html.ruby","meta.tag.block.any.html"]
-          expect(tokens[3]).toEqual value: 'class', scopes: ["text.html.ruby","meta.tag.block.any.html", "entity.other.attribute-name.html"]
-          expect(tokens[4]).toEqual value: '=', scopes: ["text.html.ruby","meta.tag.block.any.html"]
-          expect(tokens[5]).toEqual value: '\'', scopes: ["text.html.ruby","meta.tag.block.any.html","string.quoted.single.html","punctuation.definition.string.begin.html"]
-          expect(tokens[6]).toEqual value: 'name', scopes: ["text.html.ruby","meta.tag.block.any.html","string.quoted.single.html"]
-          expect(tokens[7]).toEqual value: '\'', scopes: ["text.html.ruby","meta.tag.block.any.html","string.quoted.single.html","punctuation.definition.string.end.html"]
-          expect(tokens[8]).toEqual value: '>', scopes: ["text.html.ruby","meta.tag.block.any.html","punctuation.definition.tag.end.html"]
-          expect(tokens[9]).toEqual value: '<%=', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","punctuation.section.embedded.ruby"]
-          expect(tokens[10]).toEqual value: ' ', scopes: ["text.html.ruby","source.ruby.rails.embedded.html"]
-          expect(tokens[11]).toEqual value: 'User', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","support.class.ruby"]
-          expect(tokens[12]).toEqual value: '.', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","punctuation.separator.method.ruby"]
-          expect(tokens[13]).toEqual value: 'find', scopes: ["text.html.ruby","source.ruby.rails.embedded.html"]
-          expect(tokens[14]).toEqual value: '(', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","punctuation.section.function.ruby"]
-          expect(tokens[15]).toEqual value: '2', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","constant.numeric.ruby"]
-          expect(tokens[16]).toEqual value: ')', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","punctuation.section.function.ruby"]
-          expect(tokens[17]).toEqual value: '.', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","punctuation.separator.method.ruby"]
-          expect(tokens[18]).toEqual value: 'full_name ', scopes: ["text.html.ruby","source.ruby.rails.embedded.html"]
-          expect(tokens[19]).toEqual value: '%>', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","punctuation.section.embedded.ruby"]
-          expect(tokens[20]).toEqual value: '</', scopes: ["text.html.ruby","meta.tag.block.any.html","punctuation.definition.tag.begin.html"]
-          expect(tokens[21]).toEqual value: 'div', scopes: ["text.html.ruby","meta.tag.block.any.html","entity.name.tag.block.any.html"]
-          expect(tokens[22]).toEqual value: '>', scopes: ["text.html.ruby","meta.tag.block.any.html","punctuation.definition.tag.end.html"]
+            expect(tokens[0]).toEqual value: '<', scopes: ["text.html.ruby","meta.tag.block.any.html","punctuation.definition.tag.begin.html"]
+            expect(tokens[1]).toEqual value: 'div', scopes: ["text.html.ruby","meta.tag.block.any.html","entity.name.tag.block.any.html"]
+            expect(tokens[2]).toEqual value: ' ', scopes: ["text.html.ruby","meta.tag.block.any.html"]
+            expect(tokens[3]).toEqual value: 'class', scopes: ["text.html.ruby","meta.tag.block.any.html", "entity.other.attribute-name.html"]
+            expect(tokens[4]).toEqual value: '=', scopes: ["text.html.ruby","meta.tag.block.any.html"]
+            expect(tokens[5]).toEqual value: '\'', scopes: ["text.html.ruby","meta.tag.block.any.html","string.quoted.single.html","punctuation.definition.string.begin.html"]
+            expect(tokens[6]).toEqual value: 'name', scopes: ["text.html.ruby","meta.tag.block.any.html","string.quoted.single.html"]
+            expect(tokens[7]).toEqual value: '\'', scopes: ["text.html.ruby","meta.tag.block.any.html","string.quoted.single.html","punctuation.definition.string.end.html"]
+            expect(tokens[8]).toEqual value: '>', scopes: ["text.html.ruby","meta.tag.block.any.html","punctuation.definition.tag.end.html"]
+            expect(tokens[9]).toEqual value: '<%=', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","punctuation.section.embedded.ruby"]
+            expect(tokens[10]).toEqual value: ' ', scopes: ["text.html.ruby","source.ruby.rails.embedded.html"]
+            expect(tokens[11]).toEqual value: 'User', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","support.class.ruby"]
+            expect(tokens[12]).toEqual value: '.', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","punctuation.separator.method.ruby"]
+            expect(tokens[13]).toEqual value: 'find', scopes: ["text.html.ruby","source.ruby.rails.embedded.html"]
+            expect(tokens[14]).toEqual value: '(', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","punctuation.section.function.ruby"]
+            expect(tokens[15]).toEqual value: '2', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","constant.numeric.ruby"]
+            expect(tokens[16]).toEqual value: ')', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","punctuation.section.function.ruby"]
+            expect(tokens[17]).toEqual value: '.', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","punctuation.separator.method.ruby"]
+            expect(tokens[18]).toEqual value: 'full_name ', scopes: ["text.html.ruby","source.ruby.rails.embedded.html"]
+            expect(tokens[19]).toEqual value: '%>', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","punctuation.section.embedded.ruby"]
+            expect(tokens[20]).toEqual value: '</', scopes: ["text.html.ruby","meta.tag.block.any.html","punctuation.definition.tag.begin.html"]
+            expect(tokens[21]).toEqual value: 'div', scopes: ["text.html.ruby","meta.tag.block.any.html","entity.name.tag.block.any.html"]
+            expect(tokens[22]).toEqual value: '>', scopes: ["text.html.ruby","meta.tag.block.any.html","punctuation.definition.tag.end.html"]
+
+        describe "when a grammar matching the desired scope is unavailable", ->
+          it "updates the grammar if a matching grammar is added later", ->
+            atom.deactivatePackage('html.tmbundle')
+            atom.activatePackage('ruby-on-rails-tmbundle', sync: true)
+
+            grammar = syntax.selectGrammar('foo.html.erb')
+            {tokens} = grammar.tokenizeLine("<div class='name'><%= User.find(2).full_name %></div>")
+            expect(tokens[0]).toEqual value: "<div class='name'>", scopes: ["text.html.ruby"]
+            expect(tokens[1]).toEqual value: '<%=', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","punctuation.section.embedded.ruby"]
+            expect(tokens[2]).toEqual value: ' ', scopes: ["text.html.ruby","source.ruby.rails.embedded.html"]
+            expect(tokens[3]).toEqual value: 'User', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","support.class.ruby"]
+
+            atom.activatePackage('html.tmbundle', sync: true)
+            {tokens} = grammar.tokenizeLine("<div class='name'><%= User.find(2).full_name %></div>")
+            expect(tokens[0]).toEqual value: '<', scopes: ["text.html.ruby","meta.tag.block.any.html","punctuation.definition.tag.begin.html"]
+            expect(tokens[1]).toEqual value: 'div', scopes: ["text.html.ruby","meta.tag.block.any.html","entity.name.tag.block.any.html"]
+            expect(tokens[2]).toEqual value: ' ', scopes: ["text.html.ruby","meta.tag.block.any.html"]
+            expect(tokens[3]).toEqual value: 'class', scopes: ["text.html.ruby","meta.tag.block.any.html", "entity.other.attribute-name.html"]
+            expect(tokens[4]).toEqual value: '=', scopes: ["text.html.ruby","meta.tag.block.any.html"]
+            expect(tokens[5]).toEqual value: '\'', scopes: ["text.html.ruby","meta.tag.block.any.html","string.quoted.single.html","punctuation.definition.string.begin.html"]
+            expect(tokens[6]).toEqual value: 'name', scopes: ["text.html.ruby","meta.tag.block.any.html","string.quoted.single.html"]
+            expect(tokens[7]).toEqual value: '\'', scopes: ["text.html.ruby","meta.tag.block.any.html","string.quoted.single.html","punctuation.definition.string.end.html"]
+            expect(tokens[8]).toEqual value: '>', scopes: ["text.html.ruby","meta.tag.block.any.html","punctuation.definition.tag.end.html"]
+            expect(tokens[9]).toEqual value: '<%=', scopes: ["text.html.ruby","source.ruby.rails.embedded.html","punctuation.section.embedded.ruby"]
+            expect(tokens[10]).toEqual value: ' ', scopes: ["text.html.ruby","source.ruby.rails.embedded.html"]
 
     it "can parse a grammar with newline characters in its regular expressions (regression)", ->
       grammar = new TextMateGrammar

src/app/null-grammar.coffee

@@ -12,3 +12,5 @@ class NullGrammar
 
   tokenizeLine: (line) ->
     { tokens: [new Token(value: line, scopes: ['null-grammar.text.plain'])] }
+
+  grammarAddedOrRemoved: -> # no op

src/app/syntax.coffee

@@ -31,13 +31,19 @@ class Syntax
     { deserializer: @constructor.name, @grammarOverridesByPath }
 
   addGrammar: (grammar) ->
+    previousGrammars = new Array(@grammars...)
     @grammars.push(grammar)
     @grammarsByScopeName[grammar.scopeName] = grammar
+    @notifyOtherGrammars(previousGrammars, grammar.scopeName)
     @trigger 'grammar-added', grammar
 
   removeGrammar: (grammar) ->
     _.remove(@grammars, grammar)
     delete @grammarsByScopeName[grammar.scopeName]
+    @notifyOtherGrammars(@grammars, grammar.scopeName)
+
+  notifyOtherGrammars: (grammars, scopeName) ->
+    grammar.grammarAddedOrRemoved(scopeName) for grammar in grammars
 
   setGrammarOverrideForPath: (path, scopeName) ->
     @grammarOverridesByPath[path] = scopeName

src/app/text-mate-grammar.coffee

@@ -4,6 +4,7 @@ plist = require 'plist'
 Token = require 'token'
 {OnigRegExp, OnigScanner} = require 'oniguruma'
 nodePath = require 'path'
+EventEmitter = require 'event-emitter'
 pathSplitRegex = new RegExp("[#{nodePath.sep}.]")
 TextMateScopeSelector = require 'text-mate-scope-selector'
 
@@ -27,23 +28,46 @@ class TextMateGrammar
     new TextMateGrammar(fsUtils.readObject(path))
 
   name: null
+  rawPatterns: null
+  rawRepository: null
   fileTypes: null
   scopeName: null
   repository: null
   initialRule: null
   firstLineRegex: null
+  includedGrammarScopes: null
   maxTokensPerLine: 100
 
   constructor: ({ @name, @fileTypes, @scopeName, injections, patterns, repository, @foldingStopMarker, firstLineMatch}) ->
+    @rawPatterns = patterns
+    @rawRepository = repository
     @injections = new Injections(this, injections)
-    @initialRule = new Rule(this, {@scopeName, patterns})
-    @repository = {}
     @firstLineRegex = new OnigRegExp(firstLineMatch) if firstLineMatch
     @fileTypes ?= []
+    @includedGrammarScopes = []
 
-    for name, data of repository
-      data = {patterns: [data], tempName: name} if data.begin? or data.match?
-      @repository[name] = new Rule(this, data)
+  clearRules: ->
+    @initialRule = null
+    @repository = null
+
+  getInitialRule: ->
+    @initialRule ?= new Rule(this, {@scopeName, patterns: @rawPatterns})
+
+  getRepository: ->
+    @repository ?= do =>
+      repository = {}
+      for name, data of @rawRepository
+        data = {patterns: [data], tempName: name} if data.begin? or data.match?
+        repository[name] = new Rule(this, data)
+      repository
+
+  addIncludedGrammarScope: (scope) ->
+    @includedGrammarScopes.push(scope) unless _.include(@includedGrammarScopes, scope)
+
+  grammarAddedOrRemoved: (scopeName) =>
+    return unless _.include(@includedGrammarScopes, scopeName)
+    @clearRules()
+    @trigger 'grammar-updated'
 
   getScore: (path, contents) ->
     contents = fsUtils.read(path) if not contents? and fsUtils.isFile(path)
@@ -82,7 +106,7 @@ class TextMateGrammar
       pathSuffix = pathComponents[-fileTypeComponents.length..-1]
       _.isEqual(pathSuffix, fileTypeComponents)
 
-  tokenizeLine: (line, ruleStack=[@initialRule], firstLine=false) ->
+  tokenizeLine: (line, ruleStack=[@getInitialRule()], firstLine=false) ->
     originalRuleStack = ruleStack
     ruleStack = new Array(ruleStack...) # clone ruleStack
     tokens = []
@@ -173,6 +197,8 @@ class Injections
         scanners.push(scanner)
     scanners
 
+_.extend TextMateGrammar.prototype, EventEmitter
+
 class Rule
   grammar: null
   scopeName: null
@@ -359,13 +385,14 @@ class Pattern
 
   ruleForInclude: (baseGrammar, name) ->
     if name[0] == "#"
-      @grammar.repository[name[1..]]
+      @grammar.getRepository()[name[1..]]
     else if name == "$self"
-      @grammar.initialRule
+      @grammar.getInitialRule()
     else if name == "$base"
-      baseGrammar.initialRule
+      baseGrammar.getInitialRule()
     else
-      syntax.grammarForScopeName(name)?.initialRule
+      @grammar.addIncludedGrammarScope(name)
+      syntax.grammarForScopeName(name)?.getInitialRule()
 
   getIncludedPatterns: (baseGrammar, included) ->
     if @include