diff --git a/src/token.coffee b/src/token.coffee index 8aa4a8706..583e25e9b 100644 --- a/src/token.coffee +++ b/src/token.coffee @@ -20,7 +20,11 @@ class Token firstTrailingWhitespaceIndex: null hasInvisibleCharacters: false - constructor: ({@value, @scopes, @isAtomic, @bufferDelta, @isHardTab, @hasPairedCharacter, @isSoftWrapIndentation}) -> + constructor: (properties) -> + {@value, @scopes, @isAtomic, @bufferDelta, @isHardTab, @hasPairedCharacter, @isSoftWrapIndentation} = properties + @firstNonWhitespaceIndex = properties.firstNonWhitespaceIndex ? null + @firstTrailingWhitespaceIndex = properties.firstTrailingWhitespaceIndex ? null + @screenDelta = @value.length @bufferDelta ?= @screenDelta @hasPairedCharacter ?= textUtils.hasPairedCharacter(@value) diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index 6f86847de..29ed6b88a 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -122,6 +122,8 @@ class TokenizedLine @text = text Object.defineProperty @prototype, 'tokens', get: -> + offset = 0 + atom.grammars.decodeContent @text, @tags, @parentScopes.slice(), (tokenProperties, index) => switch @specialTokens[index] when SoftTab @@ -133,6 +135,16 @@ class TokenizedLine tokenProperties.isAtomic = true tokenProperties.hasPairedCharacter = true + if offset < @firstNonWhitespaceIndex + tokenProperties.firstNonWhitespaceIndex = + Math.min(offset + tokenProperties.value.length, @firstNonWhitespaceIndex - offset) + + if @lineEnding? and (offset + tokenProperties.value.length > @firstTrailingWhitespaceIndex) + tokenProperties.firstTrailingWhitespaceIndex = + Math.max(0, @firstTrailingWhitespaceIndex - offset) + + offset += tokenProperties.value.length + new Token(tokenProperties) buildText: -> @@ -332,17 +344,8 @@ class TokenizedLine @firstNonWhitespaceIndex = @text.search(NonWhitespaceRegex) if @firstNonWhitespaceIndex > 0 and isPairedCharacter(@text, @firstNonWhitespaceIndex - 1) @firstNonWhitespaceIndex-- - firstTrailingWhitespaceIndex = @text.search(TrailingWhitespaceRegex) - @lineIsWhitespaceOnly = firstTrailingWhitespaceIndex is 0 - index = 0 - for token in @tokens - if index < @firstNonWhitespaceIndex - token.firstNonWhitespaceIndex = Math.min(index + token.value.length, @firstNonWhitespaceIndex - index) - # Only the *last* segment of a soft-wrapped line can have trailing whitespace - if @lineEnding? and (index + token.value.length > firstTrailingWhitespaceIndex) - token.firstTrailingWhitespaceIndex = Math.max(0, firstTrailingWhitespaceIndex - index) - index += token.value.length - return + @firstTrailingWhitespaceIndex = @text.search(TrailingWhitespaceRegex) + @lineIsWhitespaceOnly = @firstTrailingWhitespaceIndex is 0 substituteInvisibleCharacters: -> invisibles = @invisibles