From 6171ea33e1f58f85b43d027be7d03418107a7ab4 Mon Sep 17 00:00:00 2001 From: Corey Johnson Date: Tue, 16 Oct 2012 15:58:11 -0700 Subject: [PATCH] Add support for TextMate grammars with newlines in their regexes --- spec/app/editor-spec.coffee | 6 +++--- spec/app/text-mate-grammar-spec.coffee | 23 +++++++++++++++++++++++ spec/app/token-spec.coffee | 2 +- spec/app/tokenized-buffer-spec.coffee | 6 +++--- src/app/text-mate-grammar.coffee | 7 ++++++- 5 files changed, 36 insertions(+), 8 deletions(-) diff --git a/spec/app/editor-spec.coffee b/spec/app/editor-spec.coffee index 7ff41560c..93fdc36a7 100644 --- a/spec/app/editor-spec.coffee +++ b/spec/app/editor-spec.coffee @@ -998,7 +998,7 @@ describe "Editor", -> line0 = editor.renderedLines.find('.line:first') span0 = line0.children('span:eq(0)') expect(span0).toMatchSelector '.source.js' - expect(span0.children('span:eq(0)')).toMatchSelector '.storage.type.js' + expect(span0.children('span:eq(0)')).toMatchSelector '.storage.modifier.js' expect(span0.children('span:eq(0)').text()).toBe 'var' span0_1 = span0.children('span:eq(1)') @@ -1023,9 +1023,9 @@ describe "Editor", -> describe "when lines are updated in the buffer", -> it "syntax highlights the updated lines", -> - expect(editor.renderedLines.find('.line:eq(0) > span:first > span:first')).toMatchSelector '.storage.type.js' + expect(editor.renderedLines.find('.line:eq(0) > span:first > span:first')).toMatchSelector '.storage.modifier.js' buffer.insert([0, 0], "q") - expect(editor.renderedLines.find('.line:eq(0) > span:first > span:first')).not.toMatchSelector '.storage.type.js' + expect(editor.renderedLines.find('.line:eq(0) > span:first > span:first')).not.toMatchSelector '.storage.modifier.js' # verify that re-highlighting can occur below the changed line buffer.insert([5,0], "/* */") diff --git a/spec/app/text-mate-grammar-spec.coffee b/spec/app/text-mate-grammar-spec.coffee index c793dd386..2ef054b0d 100644 --- a/spec/app/text-mate-grammar-spec.coffee +++ b/spec/app/text-mate-grammar-spec.coffee @@ -216,3 +216,26 @@ describe "TextMateGrammar", -> expect(tokens[18]).toEqual value: '', scopes: ["text.html.ruby","meta.tag.block.any.html","punctuation.definition.tag.end.html"] + + it "can parse a grammar with newline charachters in its regular expressions (regression)", -> + grammar = new TextMateGrammar + name: "test" + scopeName: "source.imaginaryLanguage" + repository: {} + patterns: [ + { + name: "comment-body"; + begin: "//"; + end: "\\n"; + beginCaptures: + "0": { name: "comment-start" } + } + ] + + {tokens, stack} = grammar.getLineTokens("// a singleLineComment") + expect(stack.length).toBe 1 + expect(stack[0].scopeName).toBe "source.imaginaryLanguage" + + expect(tokens.length).toBe 2 + expect(tokens[0].value).toBe "//" + expect(tokens[1].value).toBe " a singleLineComment" diff --git a/spec/app/token-spec.coffee b/spec/app/token-spec.coffee index f00741779..08758bf91 100644 --- a/spec/app/token-spec.coffee +++ b/spec/app/token-spec.coffee @@ -13,4 +13,4 @@ describe "Token", -> token = _.last(screenLine.tokens) afterEach -> - editSession.destroy() \ No newline at end of file + editSession.destroy() diff --git a/spec/app/tokenized-buffer-spec.coffee b/spec/app/tokenized-buffer-spec.coffee index 8eec05c09..c3cd3dd1a 100644 --- a/spec/app/tokenized-buffer-spec.coffee +++ b/spec/app/tokenized-buffer-spec.coffee @@ -23,7 +23,7 @@ describe "TokenizedBuffer", -> describe "tokenization", -> it "tokenizes all the lines in the buffer on construction", -> - expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.type.js']) + expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.modifier.js']) expect(tokenizedBuffer.lineForScreenRow(11).tokens[1]).toEqual(value: 'return', scopes: ['source.js', 'keyword.control.js']) describe "when the buffer changes", -> @@ -77,7 +77,7 @@ describe "TokenizedBuffer", -> buffer.change(range, "foo()") # previous line 0 remains - expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.type.js']) + expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.modifier.js']) # previous line 3 should be combined with input to form line 1 expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js']) @@ -113,7 +113,7 @@ describe "TokenizedBuffer", -> buffer.change(range, "foo()\nbar()\nbaz()\nquux()") # previous line 0 remains - expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual( value: 'var', scopes: ['source.js', 'storage.type.js']) + expect(tokenizedBuffer.lineForScreenRow(0).tokens[0]).toEqual( value: 'var', scopes: ['source.js', 'storage.modifier.js']) # 3 new lines inserted expect(tokenizedBuffer.lineForScreenRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js']) diff --git a/src/app/text-mate-grammar.coffee b/src/app/text-mate-grammar.coffee index 590cfb206..8d4819123 100644 --- a/src/app/text-mate-grammar.coffee +++ b/src/app/text-mate-grammar.coffee @@ -93,9 +93,14 @@ class Rule getNextTokens: (stack, line, position) -> patterns = @getIncludedPatterns() - return null unless result = @getScanner().findNextMatch(line, position) + return null unless result = @getScanner().findNextMatch(line + "\n", position) { index, captureIndices } = result + lineLength = line.length + captureIndices = captureIndices.map (value, index) -> + value = lineLength if index % 3 != 0 and value > lineLength + value + [firstCaptureIndex, firstCaptureStart, firstCaptureEnd] = captureIndices nextTokens = patterns[index].handleMatch(stack, line, captureIndices) { nextTokens, tokensStartPosition: firstCaptureStart, tokensEndPosition: firstCaptureEnd }