Use the earliest result when both injection and non-injection patterns match

spec/app/tokenized-buffer-spec.coffee

@@ -446,6 +446,7 @@ describe "TokenizedBuffer", ->
 
   describe "when the grammar has injections", ->
     beforeEach ->
+      atom.activatePackage('html.tmbundle', sync: true)
       atom.activatePackage('php.tmbundle', sync: true)
       editSession = project.buildEditSession('hello.php', autoIndent: false)
       tokenizedBuffer = editSession.displayBuffer.tokenizedBuffer
@@ -456,14 +457,25 @@ describe "TokenizedBuffer", ->
       editSession.destroy()
 
     it "correctly includes the injected patterns when tokenizing", ->
-      functionLine = tokenizedBuffer.lineForScreenRow(0)
-      { tokens } = functionLine
+      { tokens } = tokenizedBuffer.lineForScreenRow(0)
 
-      expect(tokens[0].value).toBe "<?php"
-      expect(tokens[0].scopes).toEqual ["text.html.php", "meta.embedded.line.php", "punctuation.section.embedded.begin.php"]
+      expect(tokens[3].value).toBe "<?php"
+      expect(tokens[3].scopes).toEqual ["text.html.php", "meta.embedded.line.php", "punctuation.section.embedded.begin.php"]
 
-      expect(tokens[2].value).toBe "function"
-      expect(tokens[2].scopes).toEqual ["text.html.php", "meta.embedded.line.php", "source.php", "meta.function.php", "storage.type.function.php"]
+      expect(tokens[5].value).toBe "function"
+      expect(tokens[5].scopes).toEqual ["text.html.php", "meta.embedded.line.php", "source.php", "meta.function.php", "storage.type.function.php"]
 
-      expect(tokens[4].value).toBe "hello"
-      expect(tokens[4].scopes).toEqual ["text.html.php", "meta.embedded.line.php", "source.php", "meta.function.php", "entity.name.function.php"]
+      expect(tokens[7].value).toBe "hello"
+      expect(tokens[7].scopes).toEqual ["text.html.php", "meta.embedded.line.php", "source.php", "meta.function.php", "entity.name.function.php"]
+
+      expect(tokens[14].value).toBe "?"
+      expect(tokens[14].scopes).toEqual ["text.html.php", "meta.embedded.line.php", "source.php", "punctuation.section.embedded.end.php", "source.php"]
+
+      expect(tokens[15].value).toBe ">"
+      expect(tokens[15].scopes).toEqual ["text.html.php", "meta.embedded.line.php", "punctuation.section.embedded.end.php"]
+
+      expect(tokens[16].value).toBe "</"
+      expect(tokens[16].scopes).toEqual ["text.html.php", "meta.tag.block.any.html", "punctuation.definition.tag.begin.html"]
+
+      expect(tokens[17].value).toBe "div"
+      expect(tokens[17].scopes).toEqual ["text.html.php", "meta.tag.block.any.html", "entity.name.tag.block.any.html"]

spec/fixtures/hello.php

@@ -1 +1 @@
-<?php function hello() {} ?>
+<div><?php function hello() {} ?></div>

src/app/text-mate-grammar.coffee

@@ -35,8 +35,8 @@ class TextMateGrammar
   maxTokensPerLine: 100
 
   constructor: ({ @name, @fileTypes, @scopeName, injections, patterns, repository, @foldingStopMarker, firstLineMatch}) ->
-    injections = new Injections(this, injections)
-    @initialRule = new Rule(this, {@scopeName, patterns, injections})
+    @injections = new Injections(this, injections)
+    @initialRule = new Rule(this, {@scopeName, patterns})
     @repository = {}
     @firstLineRegex = new OnigRegExp(firstLineMatch) if firstLineMatch
     @fileTypes ?= []
@@ -181,7 +181,7 @@ class Rule
   createEndPattern: null
   anchorPosition: -1
 
-  constructor: (@grammar, {@scopeName, patterns, @injections, @endPattern}) ->
+  constructor: (@grammar, {@scopeName, patterns, @endPattern}) ->
     patterns ?= []
     @patterns = patterns.map (pattern) => new Pattern(grammar, pattern)
     @patterns.unshift(@endPattern) if @endPattern and !@endPattern.hasBackReferences
@@ -218,30 +218,46 @@ class Rule
     scanner
 
   scanInjections: (ruleStack, line, position, firstLine) ->
-    if @injections?
-      scanners = @injections.getScanners(ruleStack, position, firstLine, @anchorPosition)
+    baseGrammar = ruleStack[0].grammar
+    if injections = baseGrammar.injections
+      scanners = injections.getScanners(ruleStack, position, firstLine, @anchorPosition)
       for scanner in scanners
         result = scanner.findNextMatch(line, position)
-        return {scanner, result} if result?
-    {}
+        result?.scanner = scanner
+        return result if result?
 
-  getNextTokens: (ruleStack, line, position, firstLine) ->
-    # Add a `\n` to appease patterns that contain '\n' explicitly
+  normalizeCaptureIndices: (line, captureIndices) ->
+    lineLength = line.length
+    for value, index in captureIndices
+      if index % 3 isnt 0 and value > lineLength
+        captureIndices[index] = lineLength
+
+  findNextMatch: (ruleStack, line, position, firstLine) ->
     lineWithNewline = "#{line}\n"
     baseGrammar = ruleStack[0].grammar
 
     scanner = @getScanner(baseGrammar, position, firstLine)
     result = scanner.findNextMatch(lineWithNewline, position)
-    unless result?
-      {scanner, result} = @scanInjections(ruleStack, lineWithNewline, position, firstLine)
-    return null unless result?
-    { index, captureIndices } = result
-    # Since the `\n' (added above) is not part of the line, truncate captures to the line's actual length
-    lineLength = line.length
-    captureIndices = captureIndices.map (value, index) ->
-      value = lineLength if index % 3 != 0 and value > lineLength
-      value
+    result?.scanner = scanner
+    @normalizeCaptureIndices(line, result.captureIndices) if result?
 
+    injectionResult = @scanInjections(ruleStack, lineWithNewline, position, firstLine)
+    @normalizeCaptureIndices(line, injectionResult.captureIndices) if injectionResult?
+
+    if result? and injectionResult?
+      resultStartPosition = result.captureIndices[1]
+      injectionResultStartPosition = injectionResult.captureIndices[1]
+      if injectionResultStartPosition < resultStartPosition
+        injectionResult
+      else
+        result
+    else
+      result ? injectionResult
+
+  getNextTokens: (ruleStack, line, position, firstLine) ->
+    result = @findNextMatch(ruleStack, line, position, firstLine)
+    return null unless result?
+    { index, captureIndices, scanner } = result
     [firstCaptureIndex, firstCaptureStart, firstCaptureEnd] = captureIndices
     nextTokens = scanner.patterns[index].handleMatch(ruleStack, line, captureIndices)
     { nextTokens, tokensStartPosition: firstCaptureStart, tokensEndPosition: firstCaptureEnd }