From b37468871c1e5009dc4c6723391fd74322587ea8 Mon Sep 17 00:00:00 2001
From: Kevin Sawicki & Nathan Sobo <kevin+nathan@github.com>
Date: Thu, 18 Apr 2013 16:54:38 -0700
Subject: [PATCH] Use TextMateGrammar.tokenizeLine() to create capture rule
 tokens

---
 src/app/text-mate-grammar.coffee | 28 +++++++---------------------
 1 file changed, 7 insertions(+), 21 deletions(-)

diff --git a/src/app/text-mate-grammar.coffee b/src/app/text-mate-grammar.coffee
index dc4dc8183..a3601a1e2 100644
--- a/src/app/text-mate-grammar.coffee
+++ b/src/app/text-mate-grammar.coffee
@@ -398,24 +398,10 @@ class Pattern
 
   getTokensForCaptureRule: (rule, line, captureStart, captureEnd, scopes, stack) ->
     line = line.substring(captureStart, captureEnd)
-    lineLength = line.length
-    position = 0
-    stack = [stack..., rule]
-    tokens = []
-    while position < lineLength
-      nextTokens = rule.getNextTokens(stack, line, position, false)
-      break unless nextTokens?
-      break if nextTokens.tokensEndPosition <= position
-      if nextTokens.tokensStartPosition > position
-        tokens.push(new Token(
-          value: line[position...nextTokens.tokensStartPosition]
-          scopes: scopes
-        ))
-      position = nextTokens.tokensEndPosition
-      tokens.push(nextTokens.nextTokens...)
-
-    {tokens, tokensEndPosition: captureStart + position}
-
+    {tokens} = rule.grammar.tokenizeLine(line, [stack..., rule])
+    matchLength = 0
+    matchLength += token.value.length for token in tokens
+    {captureTokens: tokens, matchEndPosition: captureStart + matchLength}
 
   getTokensForCaptureIndices: (line, captureIndices, scopes, stack) ->
     [parentCaptureIndex, parentCaptureStart, parentCaptureEnd] = shiftCapture(captureIndices)
@@ -425,9 +411,9 @@ class Pattern
       scopes = scopes.concat(scope)
 
     if captureRule = @captures[parentCaptureIndex]?.rule
-      ruleTokens = @getTokensForCaptureRule(captureRule, line, parentCaptureStart, parentCaptureEnd, scopes, stack)
-      tokens.push(ruleTokens.tokens...)
-      parentCaptureStart = ruleTokens.tokensEndPosition
+      {captureTokens, matchEndPosition} = @getTokensForCaptureRule(captureRule, line, parentCaptureStart, parentCaptureEnd, scopes, stack)
+      tokens.push(captureTokens...)
+      parentCaptureStart = matchEndPosition
 
     previousChildCaptureEnd = parentCaptureStart
     while captureIndices.length and captureIndices[1] < parentCaptureEnd