From 15d7b2f1590779ab4c8b105ce4dcdce76dfc26db Mon Sep 17 00:00:00 2001 From: Kevin Sawicki Date: Fri, 21 Dec 2012 14:58:33 -0800 Subject: [PATCH 1/5] Add failing spec --- spec/app/tokenized-buffer-spec.coffee | 19 +++++++++++++++++++ spec/fixtures/function.mm | 4 ++++ 2 files changed, 23 insertions(+) create mode 100644 spec/fixtures/function.mm diff --git a/spec/app/tokenized-buffer-spec.coffee b/spec/app/tokenized-buffer-spec.coffee index 2ba958353..c19be4751 100644 --- a/spec/app/tokenized-buffer-spec.coffee +++ b/spec/app/tokenized-buffer-spec.coffee @@ -400,3 +400,22 @@ describe "TokenizedBuffer", -> expect(tokenizedBuffer.lineForScreenRow(1).text).toBe ' "b" => "c",' expect(tokenizedBuffer.lineForScreenRow(2).text).toBe '}' expect(tokenizedBuffer.lineForScreenRow(3).text).toBe '' + + fdescribe "when an Objective-C source file is tokenized", -> + beforeEach -> + editSession = fixturesProject.buildEditSessionForPath('function.mm', autoIndent: false) + buffer = editSession.buffer + tokenizedBuffer = editSession.displayBuffer.tokenizedBuffer + editSession.setVisible(true) + fullyTokenize(tokenizedBuffer) + + afterEach -> + editSession.destroy() + + it "correctly parses variable type when it is a built-in Cocoa class", -> + commentLine = tokenizedBuffer.lineForScreenRow(1) + expect(commentLine.text).toBe 'NSString *a = @"a\\nb";' + { tokens } = commentLine + + expect(tokens[0].value).toBe "NSString" + expect(tokens[0].scopes).toEqual ["source.objc++", "meta.function.c", "meta.block.c", "support.class.cocoa"] diff --git a/spec/fixtures/function.mm b/spec/fixtures/function.mm new file mode 100644 index 000000000..98539b972 --- /dev/null +++ b/spec/fixtures/function.mm @@ -0,0 +1,4 @@ +NSString *test() { +NSString *a = @"a\nb"; +return a; +} From 0f71848b2b99aa90af6d38603978aa8b14f8c4ac Mon Sep 17 00:00:00 2001 From: Kevin Sawicki Date: Sat, 22 Dec 2012 12:40:47 -0800 Subject: [PATCH 2/5] Add spec for semicolon at end of line --- spec/app/tokenized-buffer-spec.coffee | 10 ++++++++++ spec/fixtures/function.mm | 3 +-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/spec/app/tokenized-buffer-spec.coffee b/spec/app/tokenized-buffer-spec.coffee index c19be4751..58063d4ca 100644 --- a/spec/app/tokenized-buffer-spec.coffee +++ b/spec/app/tokenized-buffer-spec.coffee @@ -419,3 +419,13 @@ describe "TokenizedBuffer", -> expect(tokens[0].value).toBe "NSString" expect(tokens[0].scopes).toEqual ["source.objc++", "meta.function.c", "meta.block.c", "support.class.cocoa"] + + it "correctly parses the semicolon at the end of the line", -> + commentLine = tokenizedBuffer.lineForScreenRow(1) + expect(commentLine.text).toBe 'NSString *a = @"a\\nb";' + { tokens } = commentLine + + lastToken = tokens.length - 1 + expect(lastToken).toBeGreaterThan 0 + expect(tokens[lastToken].value).toBe ";" + expect(tokens[lastToken].scopes).toEqual ["source.objc++", "meta.function.c", "meta.block.c"] diff --git a/spec/fixtures/function.mm b/spec/fixtures/function.mm index 98539b972..9d81fe872 100644 --- a/spec/fixtures/function.mm +++ b/spec/fixtures/function.mm @@ -1,4 +1,3 @@ -NSString *test() { +void test() { NSString *a = @"a\nb"; -return a; } From 88ff5cac29bcca1364d3cbf24d9bfeac8faa982b Mon Sep 17 00:00:00 2001 From: Kevin Sawicki Date: Sat, 22 Dec 2012 13:04:51 -0800 Subject: [PATCH 3/5] Add spec for string characters before escape character --- spec/app/tokenized-buffer-spec.coffee | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/spec/app/tokenized-buffer-spec.coffee b/spec/app/tokenized-buffer-spec.coffee index 58063d4ca..1c84641d7 100644 --- a/spec/app/tokenized-buffer-spec.coffee +++ b/spec/app/tokenized-buffer-spec.coffee @@ -429,3 +429,11 @@ describe "TokenizedBuffer", -> expect(lastToken).toBeGreaterThan 0 expect(tokens[lastToken].value).toBe ";" expect(tokens[lastToken].scopes).toEqual ["source.objc++", "meta.function.c", "meta.block.c"] + + it "correctly parses the string characters before the escaped character", -> + commentLine = tokenizedBuffer.lineForScreenRow(1) + expect(commentLine.text).toBe 'NSString *a = @"a\\nb";' + { tokens } = commentLine + + expect(tokens[2].value).toBe '@"' + expect(tokens[2].scopes).toEqual ["source.objc++", "meta.function.c", "meta.block.c", "string.quoted.double.objc", "punctuation.definition.string.begin.objc"] From 255df32f4d3a4e6df0117aa473f531d60b749c2d Mon Sep 17 00:00:00 2001 From: Corey Johnson Date: Mon, 7 Jan 2013 14:19:58 -0800 Subject: [PATCH 4/5] spelling --- spec/app/text-mate-grammar-spec.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/app/text-mate-grammar-spec.coffee b/spec/app/text-mate-grammar-spec.coffee index 97e1ead87..99613d9b9 100644 --- a/spec/app/text-mate-grammar-spec.coffee +++ b/spec/app/text-mate-grammar-spec.coffee @@ -218,7 +218,7 @@ describe "TextMateGrammar", -> expect(tokens[21]).toEqual value: 'div', scopes: ["text.html.ruby","meta.tag.block.any.html","entity.name.tag.block.any.html"] expect(tokens[22]).toEqual value: '>', scopes: ["text.html.ruby","meta.tag.block.any.html","punctuation.definition.tag.end.html"] - it "can parse a grammar with newline charachters in its regular expressions (regression)", -> + it "can parse a grammar with newline characters in its regular expressions (regression)", -> grammar = new TextMateGrammar name: "test" scopeName: "source.imaginaryLanguage" From 7372ae00cd9907e78262943dae3823b8d14e2468 Mon Sep 17 00:00:00 2001 From: Corey Johnson & Nathan Sobo Date: Tue, 8 Jan 2013 10:50:23 -0800 Subject: [PATCH 5/5] Resolve $base includes correctly in TextMate grammars --- spec/app/tokenized-buffer-spec.coffee | 2 +- src/app/text-mate-grammar.coffee | 56 +++++++++++++++------------ 2 files changed, 32 insertions(+), 26 deletions(-) diff --git a/spec/app/tokenized-buffer-spec.coffee b/spec/app/tokenized-buffer-spec.coffee index 1c84641d7..944628904 100644 --- a/spec/app/tokenized-buffer-spec.coffee +++ b/spec/app/tokenized-buffer-spec.coffee @@ -401,7 +401,7 @@ describe "TokenizedBuffer", -> expect(tokenizedBuffer.lineForScreenRow(2).text).toBe '}' expect(tokenizedBuffer.lineForScreenRow(3).text).toBe '' - fdescribe "when an Objective-C source file is tokenized", -> + describe "when an Objective-C source file is tokenized", -> beforeEach -> editSession = fixturesProject.buildEditSessionForPath('function.mm', autoIndent: false) buffer = editSession.buffer diff --git a/src/app/text-mate-grammar.coffee b/src/app/text-mate-grammar.coffee index f46c5f40a..1bd96d996 100644 --- a/src/app/text-mate-grammar.coffee +++ b/src/app/text-mate-grammar.coffee @@ -66,19 +66,11 @@ class TextMateGrammar ruleStack.forEach (rule) -> rule.clearAnchorPosition() { tokens, ruleStack } - ruleForInclude: (name) -> - if name[0] == "#" - @repository[name[1..]] - else if name == "$self" or name == "$base" - @initialRule - else - syntax.grammarForScopeName(name)?.initialRule - class Rule grammar: null scopeName: null patterns: null - allPatterns: null + scannersByBaseGrammarName: null createEndPattern: null anchorPosition: -1 @@ -86,25 +78,26 @@ class Rule patterns ?= [] @patterns = patterns.map (pattern) => new Pattern(grammar, pattern) @patterns.unshift(@endPattern) if @endPattern and !@endPattern.hasBackReferences + @scannersByBaseGrammarName = {} - getIncludedPatterns: (included=[]) -> - return @allPatterns if @allPatterns + getIncludedPatterns: (baseGrammar, included=[]) -> return [] if _.include(included, this) included = included.concat([this]) - @allPatterns = [] + allPatterns = [] for pattern in @patterns - @allPatterns.push(pattern.getIncludedPatterns(included)...) - @allPatterns + allPatterns.push(pattern.getIncludedPatterns(baseGrammar, included)...) + allPatterns clearAnchorPosition: -> @anchorPosition = -1 - getScanner: (position, firstLine) -> - return @scanner if @scanner + getScanner: (baseGrammar, position, firstLine) -> + return scanner if scanner = @scannersByBaseGrammarName[baseGrammar.name] anchored = false regexes = [] - @getIncludedPatterns().forEach (pattern) => + patterns = @getIncludedPatterns(baseGrammar) + patterns.forEach (pattern) => if pattern.anchored anchored = true regex = pattern.replaceAnchor(firstLine, position, @anchorPosition) @@ -113,14 +106,17 @@ class Rule regexes.push regex if regex regexScanner = new OnigScanner(regexes) - @scanner = regexScanner unless anchored + regexScanner.patterns = patterns + @scannersByBaseGrammarName[baseGrammar.name] = regexScanner unless anchored regexScanner - getNextTokens: (stack, line, position, firstLine) -> - patterns = @getIncludedPatterns() + getNextTokens: (ruleStack, line, position, firstLine) -> + baseGrammar = ruleStack[0].grammar + patterns = @getIncludedPatterns(baseGrammar) + scanner = @getScanner(baseGrammar, position, firstLine) # Add a `\n` to appease patterns that contain '\n' explicitly - return null unless result = @getScanner(position, firstLine).findNextMatch("#{line}\n", position) + return null unless result = scanner.findNextMatch("#{line}\n", position) { index, captureIndices } = result # Since the `\n' (added above) is not part of the line, truncate captures to the line's actual length lineLength = line.length @@ -129,7 +125,7 @@ class Rule value [firstCaptureIndex, firstCaptureStart, firstCaptureEnd] = captureIndices - nextTokens = patterns[index].handleMatch(stack, line, captureIndices) + nextTokens = patterns[index].handleMatch(ruleStack, line, captureIndices) { nextTokens, tokensStartPosition: firstCaptureStart, tokensEndPosition: firstCaptureEnd } getRuleToPush: (line, beginPatternCaptureIndices) -> @@ -214,10 +210,20 @@ class Pattern new Pattern(@grammar, { hasBackReferences: false, match: resolvedMatch, @captures, @popRule }) - getIncludedPatterns: (included) -> + ruleForInclude: (baseGrammar, name) -> + if name[0] == "#" + @grammar.repository[name[1..]] + else if name == "$self" + @grammar.initialRule + else if name == "$base" + baseGrammar.initialRule + else + syntax.grammarForScopeName(name)?.initialRule + + getIncludedPatterns: (baseGrammar, included) -> if @include - rule = @grammar.ruleForInclude(@include) - rule?.getIncludedPatterns(included) ? [] + rule = @ruleForInclude(baseGrammar, @include) + rule?.getIncludedPatterns(baseGrammar, included) ? [] else [this]