diff --git a/spec/tokenized-buffer-spec.coffee b/spec/tokenized-buffer-spec.coffee index f45dfa8d5..5d538f51c 100644 --- a/spec/tokenized-buffer-spec.coffee +++ b/spec/tokenized-buffer-spec.coffee @@ -463,3 +463,40 @@ describe "TokenizedBuffer", -> expect(tokenizedBuffer.tokenForPosition([0,0]).value).toBe ' ' atom.config.set('editor.tabLength', 0) expect(tokenizedBuffer.tokenForPosition([0,0]).value).toBe ' ' + + describe "leading and trailing whitespace", -> + beforeEach -> + buffer = atom.project.bufferForPathSync('sample.js') + tokenizedBuffer = new TokenizedBuffer({buffer}) + fullyTokenize(tokenizedBuffer) + + it "sets ::hasLeadingWhitespace to true on tokens that have leading whitespace", -> + expect(tokenizedBuffer.lineForScreenRow(0).tokens[0].hasLeadingWhitespace).toBe false + expect(tokenizedBuffer.lineForScreenRow(1).tokens[0].hasLeadingWhitespace).toBe true + expect(tokenizedBuffer.lineForScreenRow(1).tokens[1].hasLeadingWhitespace).toBe false + expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].hasLeadingWhitespace).toBe true + expect(tokenizedBuffer.lineForScreenRow(2).tokens[1].hasLeadingWhitespace).toBe true + expect(tokenizedBuffer.lineForScreenRow(2).tokens[2].hasLeadingWhitespace).toBe false + + # The 4th token *has* leading whitespace, but isn't entirely whitespace + buffer.insert([5, 0], ' ') + expect(tokenizedBuffer.lineForScreenRow(5).tokens[3].hasLeadingWhitespace).toBe true + expect(tokenizedBuffer.lineForScreenRow(5).tokens[4].hasLeadingWhitespace).toBe false + + # Lines that are *only* whitespace are not considered to have leading whitespace + buffer.insert([10, 0], ' ') + expect(tokenizedBuffer.lineForScreenRow(10).tokens[0].hasLeadingWhitespace).toBe false + + it "sets ::hasTrailingWhitespace to true on tokens that have trailing whitespace", -> + buffer.insert([0, Infinity], ' ') + expect(tokenizedBuffer.lineForScreenRow(0).tokens[11].hasTrailingWhitespace).toBe false + expect(tokenizedBuffer.lineForScreenRow(0).tokens[12].hasTrailingWhitespace).toBe true + + # The last token *has* trailing whitespace, but isn't entirely whitespace + buffer.setTextInRange([[2, 39], [2, 40]], ' ') + expect(tokenizedBuffer.lineForScreenRow(2).tokens[14].hasTrailingWhitespace).toBe false + expect(tokenizedBuffer.lineForScreenRow(2).tokens[15].hasTrailingWhitespace).toBe true + + # Lines that are *only* whitespace are considered to have trailing whitespace + buffer.insert([10, 0], ' ') + expect(tokenizedBuffer.lineForScreenRow(10).tokens[0].hasTrailingWhitespace).toBe true diff --git a/src/token.coffee b/src/token.coffee index 08f382f9f..17699f8f8 100644 --- a/src/token.coffee +++ b/src/token.coffee @@ -20,6 +20,8 @@ class Token scopes: null isAtomic: null isHardTab: null + hasLeadingWhitespace: false + hasTrailingWhitespace: false constructor: ({@value, @scopes, @isAtomic, @bufferDelta, @isHardTab}) -> @screenDelta = @value.length diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index 01df41e21..32c1c281c 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -10,6 +10,7 @@ class TokenizedLine @text = _.pluck(@tokens, 'value').join('') @bufferDelta = _.sum(_.pluck(@tokens, 'bufferDelta')) @id = idCounter++ + @markLeadingAndTrailingWhitespaceTokens() copy: -> new TokenizedLine({@tokens, @lineEnding, @ruleStack, @startBufferColumn, @fold}) @@ -117,6 +118,16 @@ class TokenizedLine breakOutLeadingSoftTabs = token.isOnlyWhitespace() if breakOutLeadingSoftTabs outputTokens + markLeadingAndTrailingWhitespaceTokens: -> + firstNonWhitespacePosition = @text.search(/\S/) + firstTrailingWhitespacePosition = @text.search(/\s*$/) + lineIsWhitespaceOnly = firstTrailingWhitespacePosition is 0 + position = 0 + for token, i in @tokens + token.hasLeadingWhitespace = position < firstNonWhitespacePosition + token.hasTrailingWhitespace = position + token.value.length > firstTrailingWhitespacePosition + position += token.value.length + isComment: -> for token in @tokens continue if token.scopes.length is 1