diff --git a/spec/tokenized-buffer-spec.coffee b/spec/tokenized-buffer-spec.coffee index 41a63a562..0adcd2756 100644 --- a/spec/tokenized-buffer-spec.coffee +++ b/spec/tokenized-buffer-spec.coffee @@ -318,6 +318,18 @@ describe "TokenizedBuffer", -> expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[1].isAtomic).toBeFalsy() expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[1].value).toBe " current " + it "does not tokenize whitespaces followed by combining characters as leading whitespace", -> + buffer.setText(" \u030b") + fullyTokenize(tokenizedBuffer) + + {tokens} = tokenizedBuffer.tokenizedLineForRow(0) + expect(tokens[0].value).toBe " " + expect(tokens[0].hasLeadingWhitespace()).toBe true + expect(tokens[1].value).toBe " " + expect(tokens[1].hasLeadingWhitespace()).toBe true + expect(tokens[2].value).toBe " \u030b" + expect(tokens[2].hasLeadingWhitespace()).toBe false + describe "when the buffer contains hard-tabs", -> beforeEach -> waitsForPromise -> diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index d1576ab9a..903a03300 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -1,4 +1,5 @@ _ = require 'underscore-plus' +{isPairedCharacter} = require './text-utils' NonWhitespaceRegex = /\S/ LeadingWhitespaceRegex = /^\s*/ @@ -147,6 +148,8 @@ class TokenizedLine markLeadingAndTrailingWhitespaceTokens: -> firstNonWhitespaceIndex = @text.search(NonWhitespaceRegex) + if firstNonWhitespaceIndex > 0 and isPairedCharacter(@text, firstNonWhitespaceIndex - 1) + firstNonWhitespaceIndex-- firstTrailingWhitespaceIndex = @text.search(TrailingWhitespaceRegex) @lineIsWhitespaceOnly = firstTrailingWhitespaceIndex is 0 index = 0