github/atom
1
1
Fork 0
mirror of https://github.com/atom/atom.git synced 2026-04-28 03:01:47 -04:00
Code Issues Packages Projects Releases Wiki Activity

Assign ::firstNonWhitespace/TrailingWhitespaceIndex in Token::split

Browse Source 
Fixes #3277
This commit is contained in:
Nathan Sobo
2014-08-15 12:49:03 -06:00
parent 5975548cec
commit 6c52bcf20c
5 changed files with 52 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
spec/tokenized-buffer-spec.coffee
View File

@@ -629,53 +629,37 @@ describe "TokenizedBuffer", ->
tokenizedBuffer = new TokenizedBuffer({buffer})
fullyTokenize(tokenizedBuffer)
it "sets ::hasLeadingWhitespace to true and assigns ::firstNonWhitespaceIndex on tokens that have leading whitespace", ->
expect(tokenizedBuffer.lineForScreenRow(0).tokens[0].hasLeadingWhitespace).toBe false
it "assigns ::firstNonWhitespaceIndex on tokens that have leading whitespace", ->
expect(tokenizedBuffer.lineForScreenRow(0).tokens[0].firstNonWhitespaceIndex).toBe null
expect(tokenizedBuffer.lineForScreenRow(1).tokens[0].hasLeadingWhitespace).toBe true
expect(tokenizedBuffer.lineForScreenRow(1).tokens[0].firstNonWhitespaceIndex).toBe 2
expect(tokenizedBuffer.lineForScreenRow(1).tokens[1].hasLeadingWhitespace).toBe false
expect(tokenizedBuffer.lineForScreenRow(1).tokens[1].firstNonWhitespaceIndex).toBe null
expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].hasLeadingWhitespace).toBe true
expect(tokenizedBuffer.lineForScreenRow(2).tokens[0].firstNonWhitespaceIndex).toBe 2
expect(tokenizedBuffer.lineForScreenRow(2).tokens[1].hasLeadingWhitespace).toBe true
expect(tokenizedBuffer.lineForScreenRow(2).tokens[1].firstNonWhitespaceIndex).toBe 2
expect(tokenizedBuffer.lineForScreenRow(2).tokens[2].hasLeadingWhitespace).toBe false
expect(tokenizedBuffer.lineForScreenRow(2).tokens[2].firstNonWhitespaceIndex).toBe null
# The 4th token *has* leading whitespace, but isn't entirely whitespace
buffer.insert([5, 0], ' ')
expect(tokenizedBuffer.lineForScreenRow(5).tokens[3].hasLeadingWhitespace).toBe true
expect(tokenizedBuffer.lineForScreenRow(5).tokens[3].firstNonWhitespaceIndex).toBe 1
expect(tokenizedBuffer.lineForScreenRow(5).tokens[4].hasLeadingWhitespace).toBe false
expect(tokenizedBuffer.lineForScreenRow(5).tokens[4].firstNonWhitespaceIndex).toBe null
# Lines that are *only* whitespace are not considered to have leading whitespace
buffer.insert([10, 0], ' ')
expect(tokenizedBuffer.lineForScreenRow(10).tokens[0].hasLeadingWhitespace).toBe false
expect(tokenizedBuffer.lineForScreenRow(10).tokens[0].firstNonWhitespaceIndex).toBe null
it "sets ::hasTrailingWhitespace to true and assigns ::firstTrailingWhitespaceIndex on tokens that have trailing whitespace", ->
it "assigns ::firstTrailingWhitespaceIndex on tokens that have trailing whitespace", ->
buffer.insert([0, Infinity], ' ')
expect(tokenizedBuffer.lineForScreenRow(0).tokens[11].hasTrailingWhitespace).toBe false
expect(tokenizedBuffer.lineForScreenRow(0).tokens[11].firstTrailingWhitespaceIndex).toBe null
expect(tokenizedBuffer.lineForScreenRow(0).tokens[12].hasTrailingWhitespace).toBe true
expect(tokenizedBuffer.lineForScreenRow(0).tokens[12].firstTrailingWhitespaceIndex).toBe 0
# The last token *has* trailing whitespace, but isn't entirely whitespace
buffer.setTextInRange([[2, 39], [2, 40]], ' ')
expect(tokenizedBuffer.lineForScreenRow(2).tokens[14].hasTrailingWhitespace).toBe false
expect(tokenizedBuffer.lineForScreenRow(2).tokens[14].firstTrailingWhitespaceIndex).toBe null
expect(tokenizedBuffer.lineForScreenRow(2).tokens[15].hasTrailingWhitespace).toBe true
console.log tokenizedBuffer.lineForScreenRow(2).tokens[15]
expect(tokenizedBuffer.lineForScreenRow(2).tokens[15].firstTrailingWhitespaceIndex).toBe 6
# Lines that are *only* whitespace are considered to have trailing whitespace
buffer.insert([10, 0], ' ')
expect(tokenizedBuffer.lineForScreenRow(10).tokens[0].hasTrailingWhitespace).toBe true
expect(tokenizedBuffer.lineForScreenRow(10).tokens[0].firstTrailingWhitespaceIndex).toBe 0
it "only marks trailing whitespace on the last segment of a soft-wrapped line", ->
@@ -683,10 +667,8 @@ describe "TokenizedBuffer", ->
tokenizedLine = tokenizedBuffer.lineForScreenRow(0)
[segment1, segment2] = tokenizedLine.softWrapAt(16)
expect(segment1.tokens[5].value).toBe ' '
expect(segment1.tokens[5].hasTrailingWhitespace).toBe false
expect(segment1.tokens[5].firstTrailingWhitespaceIndex).toBe null
expect(segment2.tokens[6].value).toBe ' '
expect(segment2.tokens[6].hasTrailingWhitespace).toBe true
expect(segment2.tokens[6].firstTrailingWhitespaceIndex).toBe 0
it "sets leading and trailing whitespace correctly on a line with invisible characters that is copied", ->
@@ -696,8 +678,23 @@ describe "TokenizedBuffer", ->
fullyTokenize(tokenizedBuffer)
line = tokenizedBuffer.lineForScreenRow(0).copy()
expect(line.tokens[0].hasLeadingWhitespace).toBe true
expect(line.tokens[line.tokens.length - 1].hasTrailingWhitespace).toBe true
expect(line.tokens[0].firstNonWhitespaceIndex).toBe 2
expect(line.tokens[line.tokens.length - 1].firstTrailingWhitespaceIndex).toBe 0
it "sets the ::firstNonWhitespaceIndex and ::firstTrailingWhitespaceIndex correctly when tokens are split for soft-wrapping", ->
tokenizedBuffer.setInvisibles(space: 'S')
buffer.setText(" token ")
fullyTokenize(tokenizedBuffer)
token = tokenizedBuffer.tokenizedLines[0].tokens[0]
[leftToken, rightToken] = token.splitAt(1)
expect(leftToken.hasInvisibleCharacters).toBe true
expect(leftToken.firstNonWhitespaceIndex).toBe 1
expect(leftToken.firstTrailingWhitespaceIndex).toBe null
expect(leftToken.hasInvisibleCharacters).toBe true
expect(rightToken.firstNonWhitespaceIndex).toBe null
expect(rightToken.firstTrailingWhitespaceIndex).toBe 5
describe "indent level", ->
beforeEach ->

2
src/editor-view.coffee
View File

@@ -1501,7 +1501,7 @@ class EditorView extends View
position = 0
for token in tokens
@updateScopeStack(line, scopeStack, token.scopes)
hasIndentGuide = not mini and showIndentGuide and (token.hasLeadingWhitespace or (token.hasTrailingWhitespace and lineIsWhitespaceOnly))
hasIndentGuide = not mini and showIndentGuide and (token.hasLeadingWhitespace() or (token.hasTrailingWhitespace() and lineIsWhitespaceOnly))
line.push(token.getValueAsHtml({invisibles, hasIndentGuide}))
position += token.value.length

2
src/lines-component.coffee
View File

@@ -202,7 +202,7 @@ LinesComponent = React.createClass
lineIsWhitespaceOnly = firstTrailingWhitespacePosition is 0
for token in tokens
innerHTML += @updateScopeStack(scopeStack, token.scopes)
hasIndentGuide = not mini and showIndentGuide and (token.hasLeadingWhitespace or (token.hasTrailingWhitespace and lineIsWhitespaceOnly))
hasIndentGuide = not mini and showIndentGuide and (token.hasLeadingWhitespace() or (token.hasTrailingWhitespace() and lineIsWhitespaceOnly))
innerHTML += token.getValueAsHtml({hasIndentGuide})
innerHTML += @popScope(scopeStack) while scopeStack.length > 0

37
src/token.coffee
View File

@@ -16,8 +16,6 @@ class Token
scopes: null
isAtomic: null
isHardTab: null
hasLeadingWhitespace: false
hasTrailingWhitespace: false
firstNonWhitespaceIndex: null
firstTrailingWhitespaceIndex: null
hasInvisibleCharacters: false
@@ -34,9 +32,26 @@ class Token
/^meta\.brace\b/.test(_.last(@scopes))
splitAt: (splitIndex) ->
value1 = @value.substring(0, splitIndex)
value2 = @value.substring(splitIndex)
[new Token(value: value1, scopes: @scopes), new Token(value: value2, scopes: @scopes)]
leftToken = new Token(value: @value.substring(0, splitIndex), scopes: @scopes)
rightToken = new Token(value: @value.substring(splitIndex), scopes: @scopes)
if @firstNonWhitespaceIndex?
leftToken.firstNonWhitespaceIndex = Math.min(splitIndex, @firstNonWhitespaceIndex)
leftToken.hasInvisibleCharacters = @hasInvisibleCharacters
if @firstNonWhitespaceIndex > splitIndex
rightToken.firstNonWhitespaceIndex = @firstNonWhitespaceIndex - splitIndex
rightToken.hasInvisibleCharacters = @hasInvisibleCharacters
if @firstTrailingWhitespaceIndex?
rightToken.firstTrailingWhitespaceIndex = Math.max(0, @firstTrailingWhitespaceIndex - splitIndex)
rightToken.hasInvisibleCharacters = @hasInvisibleCharacters
if @firstTrailingWhitespaceIndex < splitIndex
leftToken.firstTrailingWhitespaceIndex = @firstTrailingWhitespaceIndex
leftToken.hasInvisibleCharacters = @hasInvisibleCharacters
[leftToken, rightToken]
whitespaceRegexForTabLength: (tabLength) ->
WhitespaceRegexesByTabLength[tabLength] ?= new RegExp("([ ]{#{tabLength}})|(\t)|([^\t]+)", "g")
@@ -148,7 +163,7 @@ class Token
leadingHtml = ''
trailingHtml = ''
if @hasLeadingWhitespace
if @hasLeadingWhitespace()
leadingWhitespace = @value.substring(0, @firstNonWhitespaceIndex)
classes = 'leading-whitespace'
@@ -158,12 +173,12 @@ class Token
leadingHtml = "<span class='#{classes}'>#{leadingWhitespace}</span>"
startIndex = @firstNonWhitespaceIndex
if @hasTrailingWhitespace
if @hasTrailingWhitespace()
tokenIsOnlyWhitespace = @firstTrailingWhitespaceIndex is 0
trailingWhitespace = @value.substring(@firstTrailingWhitespaceIndex)
classes = 'trailing-whitespace'
classes += ' indent-guide' if hasIndentGuide and not @hasLeadingWhitespace and tokenIsOnlyWhitespace
classes += ' indent-guide' if hasIndentGuide and not @hasLeadingWhitespace() and tokenIsOnlyWhitespace
classes += ' invisible-character' if @hasInvisibleCharacters
trailingHtml = "<span class='#{classes}'>#{trailingWhitespace}</span>"
@@ -198,3 +213,9 @@ class Token
when '<' then '&lt;'
when '>' then '&gt;'
else match
hasLeadingWhitespace: ->
@firstNonWhitespaceIndex? and @firstNonWhitespaceIndex > 0
hasTrailingWhitespace: ->
@firstTrailingWhitespaceIndex? and @firstTrailingWhitespaceIndex < @value.length

6
src/tokenized-line.coffee
View File

@@ -150,11 +150,9 @@ class TokenizedLine
index = 0
for token in @tokens
if index < firstNonWhitespaceIndex
token.hasLeadingWhitespace = true
token.firstNonWhitespaceIndex = Math.min(index + token.value.length, firstNonWhitespaceIndex - index)
# Only the *last* segment of a soft-wrapped line can have trailing whitespace
if @lineEnding? and (index + token.value.length > firstTrailingWhitespaceIndex)
token.hasTrailingWhitespace = true
token.firstTrailingWhitespaceIndex = Math.max(0, firstTrailingWhitespaceIndex - index)
index += token.value.length
@@ -170,12 +168,12 @@ class TokenizedLine
changedText = true
else
if invisibles.space
if token.hasLeadingWhitespace
if token.hasLeadingWhitespace()
token.value = token.value.replace LeadingWhitespaceRegex, (leadingWhitespace) ->
leadingWhitespace.replace RepeatedSpaceRegex, invisibles.space
token.hasInvisibleCharacters = true
changedText = true
if token.hasTrailingWhitespace
if token.hasTrailingWhitespace()
token.value = token.value.replace TrailingWhitespaceRegex, (leadingWhitespace) ->
leadingWhitespace.replace RepeatedSpaceRegex, invisibles.space
token.hasInvisibleCharacters = true