Merge branch 'objective-c-grammar'

spec/app/text-mate-grammar-spec.coffee

@@ -218,7 +218,7 @@ describe "TextMateGrammar", ->
           expect(tokens[21]).toEqual value: 'div', scopes: ["text.html.ruby","meta.tag.block.any.html","entity.name.tag.block.any.html"]
           expect(tokens[22]).toEqual value: '>', scopes: ["text.html.ruby","meta.tag.block.any.html","punctuation.definition.tag.end.html"]
 
-    it "can parse a grammar with newline charachters in its regular expressions (regression)", ->
+    it "can parse a grammar with newline characters in its regular expressions (regression)", ->
       grammar = new TextMateGrammar
         name: "test"
         scopeName: "source.imaginaryLanguage"

spec/app/tokenized-buffer-spec.coffee

@@ -400,3 +400,40 @@ describe "TokenizedBuffer", ->
       expect(tokenizedBuffer.lineForScreenRow(1).text).toBe '  "b" => "c",'
       expect(tokenizedBuffer.lineForScreenRow(2).text).toBe '}'
       expect(tokenizedBuffer.lineForScreenRow(3).text).toBe ''
+
+  describe "when an Objective-C source file is tokenized", ->
+    beforeEach ->
+      editSession =  fixturesProject.buildEditSessionForPath('function.mm', autoIndent: false)
+      buffer = editSession.buffer
+      tokenizedBuffer = editSession.displayBuffer.tokenizedBuffer
+      editSession.setVisible(true)
+      fullyTokenize(tokenizedBuffer)
+
+    afterEach ->
+      editSession.destroy()
+
+    it "correctly parses variable type when it is a built-in Cocoa class", ->
+      commentLine = tokenizedBuffer.lineForScreenRow(1)
+      expect(commentLine.text).toBe 'NSString *a = @"a\\nb";'
+      { tokens } = commentLine
+
+      expect(tokens[0].value).toBe "NSString"
+      expect(tokens[0].scopes).toEqual ["source.objc++", "meta.function.c", "meta.block.c", "support.class.cocoa"]
+
+    it "correctly parses the semicolon at the end of the line", ->
+      commentLine = tokenizedBuffer.lineForScreenRow(1)
+      expect(commentLine.text).toBe 'NSString *a = @"a\\nb";'
+      { tokens } = commentLine
+
+      lastToken = tokens.length - 1
+      expect(lastToken).toBeGreaterThan 0
+      expect(tokens[lastToken].value).toBe ";"
+      expect(tokens[lastToken].scopes).toEqual ["source.objc++", "meta.function.c", "meta.block.c"]
+
+    it "correctly parses the string characters before the escaped character", ->
+      commentLine = tokenizedBuffer.lineForScreenRow(1)
+      expect(commentLine.text).toBe 'NSString *a = @"a\\nb";'
+      { tokens } = commentLine
+
+      expect(tokens[2].value).toBe '@"'
+      expect(tokens[2].scopes).toEqual ["source.objc++", "meta.function.c", "meta.block.c", "string.quoted.double.objc", "punctuation.definition.string.begin.objc"]

spec/fixtures/function.mm

@@ -0,0 +1,3 @@
+void test() {
+NSString *a = @"a\nb";
+}

src/app/text-mate-grammar.coffee

@@ -66,19 +66,11 @@ class TextMateGrammar
     ruleStack.forEach (rule) -> rule.clearAnchorPosition()
     { tokens, ruleStack }
 
-  ruleForInclude: (name) ->
-    if name[0] == "#"
-      @repository[name[1..]]
-    else if name == "$self" or name == "$base"
-      @initialRule
-    else
-      syntax.grammarForScopeName(name)?.initialRule
-
 class Rule
   grammar: null
   scopeName: null
   patterns: null
-  allPatterns: null
+  scannersByBaseGrammarName: null
   createEndPattern: null
   anchorPosition: -1
 
@@ -86,25 +78,26 @@ class Rule
     patterns ?= []
     @patterns = patterns.map (pattern) => new Pattern(grammar, pattern)
     @patterns.unshift(@endPattern) if @endPattern and !@endPattern.hasBackReferences
+    @scannersByBaseGrammarName = {}
 
-  getIncludedPatterns: (included=[]) ->
-    return @allPatterns if @allPatterns
+  getIncludedPatterns: (baseGrammar, included=[]) ->
     return [] if _.include(included, this)
 
     included = included.concat([this])
-    @allPatterns = []
+    allPatterns = []
     for pattern in @patterns
-      @allPatterns.push(pattern.getIncludedPatterns(included)...)
-    @allPatterns
+      allPatterns.push(pattern.getIncludedPatterns(baseGrammar, included)...)
+    allPatterns
 
   clearAnchorPosition: -> @anchorPosition = -1
 
-  getScanner: (position, firstLine) ->
-    return @scanner if @scanner
+  getScanner: (baseGrammar, position, firstLine) ->
+    return scanner if scanner = @scannersByBaseGrammarName[baseGrammar.name]
 
     anchored = false
     regexes = []
-    @getIncludedPatterns().forEach (pattern) =>
+    patterns = @getIncludedPatterns(baseGrammar)
+    patterns.forEach (pattern) =>
       if pattern.anchored
         anchored = true
         regex = pattern.replaceAnchor(firstLine, position, @anchorPosition)
@@ -113,14 +106,17 @@ class Rule
       regexes.push regex if regex
 
     regexScanner = new OnigScanner(regexes)
-    @scanner = regexScanner unless anchored
+    regexScanner.patterns = patterns
+    @scannersByBaseGrammarName[baseGrammar.name] = regexScanner unless anchored
     regexScanner
 
-  getNextTokens: (stack, line, position, firstLine) ->
-    patterns = @getIncludedPatterns()
+  getNextTokens: (ruleStack, line, position, firstLine) ->
+    baseGrammar = ruleStack[0].grammar
+    patterns = @getIncludedPatterns(baseGrammar)
 
+    scanner = @getScanner(baseGrammar, position, firstLine)
     # Add a `\n` to appease patterns that contain '\n' explicitly
-    return null unless result = @getScanner(position, firstLine).findNextMatch("#{line}\n", position)
+    return null unless result = scanner.findNextMatch("#{line}\n", position)
     { index, captureIndices } = result
     # Since the `\n' (added above) is not part of the line, truncate captures to the line's actual length
     lineLength = line.length
@@ -129,7 +125,7 @@ class Rule
       value
 
     [firstCaptureIndex, firstCaptureStart, firstCaptureEnd] = captureIndices
-    nextTokens = patterns[index].handleMatch(stack, line, captureIndices)
+    nextTokens = patterns[index].handleMatch(ruleStack, line, captureIndices)
     { nextTokens, tokensStartPosition: firstCaptureStart, tokensEndPosition: firstCaptureEnd }
 
   getRuleToPush: (line, beginPatternCaptureIndices) ->
@@ -214,10 +210,20 @@ class Pattern
 
     new Pattern(@grammar, { hasBackReferences: false, match: resolvedMatch, @captures, @popRule })
 
-  getIncludedPatterns: (included) ->
+  ruleForInclude: (baseGrammar, name) ->
+    if name[0] == "#"
+      @grammar.repository[name[1..]]
+    else if name == "$self"
+      @grammar.initialRule
+    else if name == "$base"
+      baseGrammar.initialRule
+    else
+      syntax.grammarForScopeName(name)?.initialRule
+
+  getIncludedPatterns: (baseGrammar, included) ->
     if @include
-      rule = @grammar.ruleForInclude(@include)
-      rule?.getIncludedPatterns(included) ? []
+      rule = @ruleForInclude(baseGrammar, @include)
+      rule?.getIncludedPatterns(baseGrammar, included) ? []
     else
       [this]