From c63834924a38ceba3af3e0f10bf286b4e596f8a0 Mon Sep 17 00:00:00 2001
From: Kevin Sawicki <kevin@github.com>
Date: Thu, 18 Apr 2013 16:07:25 -0700
Subject: [PATCH] Support patterns included in captures

Previously only the capture's name was considered when processing
tokens for capture indices.

Now the capture's patterns are matched against the captured region
if they exist.
---
 spec/app/text-mate-grammar-spec.coffee | 23 +++++++++++++++
 src/app/text-mate-grammar.coffee       | 39 ++++++++++++++++++++++++--
 2 files changed, 59 insertions(+), 3 deletions(-)

diff --git a/spec/app/text-mate-grammar-spec.coffee b/spec/app/text-mate-grammar-spec.coffee
index eb9033e14..045edf9cb 100644
--- a/spec/app/text-mate-grammar-spec.coffee
+++ b/spec/app/text-mate-grammar-spec.coffee
@@ -308,3 +308,26 @@ describe "TextMateGrammar", ->
         expect(tokens.length).toBe 5
         expect(tokens[4].value).toBe "three(four(five(_param_)))))"
         expect(ruleStack).toEqual originalRuleStack
+
+    describe "when a grammar's captures has patterns", ->
+      beforeEach ->
+        atom.activatePackage('php.tmbundle', sync: true)
+
+      it "matches the patterns and includes the scope specified as the pattern's match name", ->
+        grammar = syntax.selectGrammar("hello.php")
+        {tokens} = grammar.tokenizeLine("<?php public final function meth() {} ?>")
+
+        expect(tokens[2].value).toBe "public"
+        expect(tokens[2].scopes).toEqual ["text.html.php", "meta.embedded.line.php", "source.php", "meta.function.php", "storage.modifier.php"]
+
+        expect(tokens[3].value).toBe " "
+        expect(tokens[3].scopes).toEqual ["text.html.php", "meta.embedded.line.php", "source.php", "meta.function.php"]
+
+        expect(tokens[4].value).toBe "final"
+        expect(tokens[4].scopes).toEqual ["text.html.php", "meta.embedded.line.php", "source.php", "meta.function.php", "storage.modifier.php"]
+
+        expect(tokens[5].value).toBe " "
+        expect(tokens[5].scopes).toEqual ["text.html.php", "meta.embedded.line.php", "source.php", "meta.function.php"]
+
+        expect(tokens[6].value).toBe "function"
+        expect(tokens[6].scopes).toEqual ["text.html.php", "meta.embedded.line.php", "source.php", "meta.function.php", "storage.type.function.php"]
diff --git a/src/app/text-mate-grammar.coffee b/src/app/text-mate-grammar.coffee
index f9d14fb35..dc4dc8183 100644
--- a/src/app/text-mate-grammar.coffee
+++ b/src/app/text-mate-grammar.coffee
@@ -293,6 +293,13 @@ class Pattern
       @captures = beginCaptures ? captures
       endPattern = new Pattern(@grammar, { match: end, captures: endCaptures ? captures, popRule: true})
       @pushRule = new Rule(@grammar, { @scopeName, patterns, endPattern })
+
+    if @captures?
+      for group, capture of @captures
+        if capture.patterns?.length > 0 and not capture.rule
+          capture.scopeName = @scopeName
+          capture.rule = new Rule(@grammar, capture)
+
     @anchored = @hasAnchor()
 
   getRegex: (firstLine, position, anchorPosition) ->
@@ -372,7 +379,7 @@ class Pattern
     scopes.push(@scopeName) if @scopeName and not @popRule
 
     if @captures
-      tokens = @getTokensForCaptureIndices(line, _.clone(captureIndices), scopes)
+      tokens = @getTokensForCaptureIndices(line, _.clone(captureIndices), scopes, stack)
     else
       [start, end] = captureIndices[1..2]
       zeroLengthMatch = end == start
@@ -389,13 +396,39 @@ class Pattern
 
     tokens
 
-  getTokensForCaptureIndices: (line, captureIndices, scopes) ->
+  getTokensForCaptureRule: (rule, line, captureStart, captureEnd, scopes, stack) ->
+    line = line.substring(captureStart, captureEnd)
+    lineLength = line.length
+    position = 0
+    stack = [stack..., rule]
+    tokens = []
+    while position < lineLength
+      nextTokens = rule.getNextTokens(stack, line, position, false)
+      break unless nextTokens?
+      break if nextTokens.tokensEndPosition <= position
+      if nextTokens.tokensStartPosition > position
+        tokens.push(new Token(
+          value: line[position...nextTokens.tokensStartPosition]
+          scopes: scopes
+        ))
+      position = nextTokens.tokensEndPosition
+      tokens.push(nextTokens.nextTokens...)
+
+    {tokens, tokensEndPosition: captureStart + position}
+
+
+  getTokensForCaptureIndices: (line, captureIndices, scopes, stack) ->
     [parentCaptureIndex, parentCaptureStart, parentCaptureEnd] = shiftCapture(captureIndices)
 
     tokens = []
     if scope = @captures[parentCaptureIndex]?.name
       scopes = scopes.concat(scope)
 
+    if captureRule = @captures[parentCaptureIndex]?.rule
+      ruleTokens = @getTokensForCaptureRule(captureRule, line, parentCaptureStart, parentCaptureEnd, scopes, stack)
+      tokens.push(ruleTokens.tokens...)
+      parentCaptureStart = ruleTokens.tokensEndPosition
+
     previousChildCaptureEnd = parentCaptureStart
     while captureIndices.length and captureIndices[1] < parentCaptureEnd
       [childCaptureIndex, childCaptureStart, childCaptureEnd] = captureIndices
@@ -412,7 +445,7 @@ class Pattern
           scopes: scopes
         ))
 
-      captureTokens = @getTokensForCaptureIndices(line, captureIndices, scopes)
+      captureTokens = @getTokensForCaptureIndices(line, captureIndices, scopes, stack)
       tokens.push(captureTokens...)
       previousChildCaptureEnd = childCaptureEnd