github/atom
1
1
Fork 0
mirror of https://github.com/atom/atom.git synced 2026-01-23 13:58:08 -05:00
Code Issues Packages Projects Releases Wiki Activity

Revert "Merge pull request #6757 from atom/ns-less-memory-for-tokens"

Browse Source 
This reverts commit 0cd1f110b5, reversing
changes made to d75d202d33.

Conflicts:
	package.json
This commit is contained in:
Nathan Sobo
2015-05-21 16:25:23 +02:00
parent 75d02cdbc9
commit 7cb0bc3bc2
15 changed files with 601 additions and 711 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
package.json
View File

@@ -32,7 +32,7 @@
"delegato": "^1",
"emissary": "^1.3.3",
"event-kit": "^1.2.0",
"first-mate": "^4.1.4",
"first-mate": "^3.1",
"fs-plus": "^2.8.0",
"fstream": "0.1.24",
"fuzzaldrin": "^2.1",
@@ -151,7 +151,7 @@
"language-ruby": "0.54.0",
"language-ruby-on-rails": "0.21.0",
"language-sass": "0.38.0",
"language-shellscript": "0.15.0",
"language-shellscript": "0.14.0",
"language-source": "0.9.0",
"language-sql": "0.15.0",
"language-text": "0.6.0",

36
spec/text-editor-presenter-spec.coffee
View File

@@ -670,11 +670,7 @@ describe "TextEditorPresenter", ->
expectValues lineStateForScreenRow(presenter, 4), {
screenRow: 4
text: line4.text
tags: line4.tags
specialTokens: line4.specialTokens
firstNonWhitespaceIndex: line4.firstNonWhitespaceIndex
firstTrailingWhitespaceIndex: line4.firstTrailingWhitespaceIndex
invisibles: line4.invisibles
tokens: line4.tokens
top: 10 * 4
}
@@ -682,11 +678,7 @@ describe "TextEditorPresenter", ->
expectValues lineStateForScreenRow(presenter, 5), {
screenRow: 5
text: line5.text
tags: line5.tags
specialTokens: line5.specialTokens
firstNonWhitespaceIndex: line5.firstNonWhitespaceIndex
firstTrailingWhitespaceIndex: line5.firstTrailingWhitespaceIndex
invisibles: line5.invisibles
tokens: line5.tokens
top: 10 * 5
}
@@ -694,11 +686,7 @@ describe "TextEditorPresenter", ->
expectValues lineStateForScreenRow(presenter, 6), {
screenRow: 6
text: line6.text
tags: line6.tags
specialTokens: line6.specialTokens
firstNonWhitespaceIndex: line6.firstNonWhitespaceIndex
firstTrailingWhitespaceIndex: line6.firstTrailingWhitespaceIndex
invisibles: line6.invisibles
tokens: line6.tokens
top: 10 * 6
}
@@ -706,11 +694,7 @@ describe "TextEditorPresenter", ->
expectValues lineStateForScreenRow(presenter, 7), {
screenRow: 7
text: line7.text
tags: line7.tags
specialTokens: line7.specialTokens
firstNonWhitespaceIndex: line7.firstNonWhitespaceIndex
firstTrailingWhitespaceIndex: line7.firstTrailingWhitespaceIndex
invisibles: line7.invisibles
tokens: line7.tokens
top: 10 * 7
}
@@ -718,11 +702,7 @@ describe "TextEditorPresenter", ->
expectValues lineStateForScreenRow(presenter, 8), {
screenRow: 8
text: line8.text
tags: line8.tags
specialTokens: line8.specialTokens
firstNonWhitespaceIndex: line8.firstNonWhitespaceIndex
firstTrailingWhitespaceIndex: line8.firstTrailingWhitespaceIndex
invisibles: line8.invisibles
tokens: line8.tokens
top: 10 * 8
}
@@ -817,19 +797,19 @@ describe "TextEditorPresenter", ->
line1 = editor.tokenizedLineForScreenRow(1)
expectValues lineStateForScreenRow(presenter, 1), {
text: line1.text
tags: line1.tags
tokens: line1.tokens
}
line2 = editor.tokenizedLineForScreenRow(2)
expectValues lineStateForScreenRow(presenter, 2), {
text: line2.text
tags: line2.tags
tokens: line2.tokens
}
line3 = editor.tokenizedLineForScreenRow(3)
expectValues lineStateForScreenRow(presenter, 3), {
text: line3.text
tags: line3.tags
tokens: line3.tokens
}
it "does not remove out-of-view lines corresponding to ::mouseWheelScreenRow until ::stoppedScrollingDelay elapses", ->

3
spec/text-editor-spec.coffee
View File

@@ -4110,9 +4110,8 @@ describe "TextEditor", ->
runs ->
grammar = atom.grammars.selectGrammar("text.js")
{line, tags} = grammar.tokenizeLine("var i; // http://github.com")
{tokens} = grammar.tokenizeLine("var i; // http://github.com")
tokens = atom.grammars.decodeTokens(line, tags)
expect(tokens[0].value).toBe "var"
expect(tokens[0].scopes).toEqual ["source.js", "storage.modifier.js"]

28
spec/tokenized-buffer-spec.coffee
View File

@@ -296,6 +296,14 @@ describe "TokenizedBuffer", ->
expect(tokenizedBuffer.tokenizedLineForRow(5).ruleStack?).toBeTruthy()
expect(tokenizedBuffer.tokenizedLineForRow(6).ruleStack?).toBeTruthy()
describe ".findOpeningBracket(closingBufferPosition)", ->
it "returns the position of the matching bracket, skipping any nested brackets", ->
expect(tokenizedBuffer.findOpeningBracket([9, 2])).toEqual [1, 29]
describe ".findClosingBracket(startBufferPosition)", ->
it "returns the position of the matching bracket, skipping any nested brackets", ->
expect(tokenizedBuffer.findClosingBracket([1, 29])).toEqual [9, 2]
it "tokenizes leading whitespace based on the new tab length", ->
expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[0].isAtomic).toBeTruthy()
expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[0].value).toBe " "
@@ -572,7 +580,7 @@ describe "TokenizedBuffer", ->
describe "when the selector matches a run of multiple tokens at the position", ->
it "returns the range covered by all contigous tokens (within a single line)", ->
expect(tokenizedBuffer.bufferRangeForScopeAtPosition('.meta.function', [1, 18])).toEqual [[1, 6], [1, 28]]
expect(tokenizedBuffer.bufferRangeForScopeAtPosition('.function', [1, 18])).toEqual [[1, 6], [1, 28]]
describe "when the editor.tabLength config value changes", ->
it "updates the tab length of the tokenized lines", ->
@@ -689,6 +697,22 @@ describe "TokenizedBuffer", ->
expect(line.tokens[0].firstNonWhitespaceIndex).toBe 2
expect(line.tokens[line.tokens.length - 1].firstTrailingWhitespaceIndex).toBe 0
it "sets the ::firstNonWhitespaceIndex and ::firstTrailingWhitespaceIndex correctly when tokens are split for soft-wrapping", ->
atom.config.set("editor.showInvisibles", true)
atom.config.set("editor.invisibles", space: 'S')
buffer.setText(" token ")
fullyTokenize(tokenizedBuffer)
token = tokenizedBuffer.tokenizedLines[0].tokens[0]
[leftToken, rightToken] = token.splitAt(1)
expect(leftToken.hasInvisibleCharacters).toBe true
expect(leftToken.firstNonWhitespaceIndex).toBe 1
expect(leftToken.firstTrailingWhitespaceIndex).toBe null
expect(leftToken.hasInvisibleCharacters).toBe true
expect(rightToken.firstNonWhitespaceIndex).toBe null
expect(rightToken.firstTrailingWhitespaceIndex).toBe 5
describe ".indentLevel on tokenized lines", ->
beforeEach ->
buffer = atom.project.bufferForPathSync('sample.js')
@@ -728,7 +752,7 @@ describe "TokenizedBuffer", ->
it "updates empty line indent guides when the empty line is the last line", ->
buffer.insert([12, 2], '\n')
# The newline and the tab need to be in two different operations to surface the bug
# The newline and he tab need to be in two different operations to surface the bug
buffer.insert([12, 0], ' ')
expect(tokenizedBuffer.tokenizedLineForRow(13).indentLevel).toBe 1

21
spec/tokenized-line-spec.coffee
View File

@@ -17,3 +17,24 @@ describe "TokenizedLine", ->
it "returns false when the line is not only whitespace", ->
expect(editor.tokenizedLineForScreenRow(0).isOnlyWhitespace()).toBe false
expect(editor.tokenizedLineForScreenRow(2).isOnlyWhitespace()).toBe false
describe "::getScopeTree()", ->
it "returns a tree whose inner nodes are scopeDescriptor and whose leaf nodes are tokens in those scopeDescriptor", ->
[tokens, tokenIndex] = []
ensureValidScopeTree = (scopeTree, scopeDescriptor=[]) ->
if scopeTree.children?
for child in scopeTree.children
ensureValidScopeTree(child, scopeDescriptor.concat([scopeTree.scope]))
else
expect(scopeTree).toBe tokens[tokenIndex++]
expect(scopeDescriptor).toEqual scopeTree.scopes
waitsForPromise ->
atom.project.open('coffee.coffee').then (o) -> editor = o
runs ->
tokenIndex = 0
tokens = editor.tokenizedLineForScreenRow(1).tokens
scopeTree = editor.tokenizedLineForScreenRow(1).getScopeTree()
ensureValidScopeTree(scopeTree)

32
src/display-buffer.coffee
View File

@@ -2,7 +2,6 @@ _ = require 'underscore-plus'
Serializable = require 'serializable'
{CompositeDisposable, Emitter} = require 'event-kit'
{Point, Range} = require 'text-buffer'
Grim = require 'grim'
TokenizedBuffer = require './tokenized-buffer'
RowMap = require './row-map'
Fold = require './fold'
@@ -10,6 +9,7 @@ Model = require './model'
Token = require './token'
Decoration = require './decoration'
Marker = require './marker'
Grim = require 'grim'
class BufferToScreenConversionError extends Error
constructor: (@message, @metadata) ->
@@ -659,19 +659,16 @@ class DisplayBuffer extends Model
top = targetRow * @lineHeightInPixels
left = 0
column = 0
iterator = @tokenizedLineForScreenRow(targetRow).getTokenIterator()
while iterator.next()
charWidths = @getScopedCharWidths(iterator.getScopes())
for token in @tokenizedLineForScreenRow(targetRow).tokens
charWidths = @getScopedCharWidths(token.scopes)
valueIndex = 0
value = iterator.getText()
while valueIndex < value.length
if iterator.isPairedCharacter()
char = value
while valueIndex < token.value.length
if token.hasPairedCharacter
char = token.value.substr(valueIndex, 2)
charLength = 2
valueIndex += 2
else
char = value[valueIndex]
char = token.value[valueIndex]
charLength = 1
valueIndex++
@@ -692,19 +689,16 @@ class DisplayBuffer extends Model
left = 0
column = 0
iterator = @tokenizedLineForScreenRow(row).getTokenIterator()
while iterator.next()
charWidths = @getScopedCharWidths(iterator.getScopes())
value = iterator.getText()
for token in @tokenizedLineForScreenRow(row).tokens
charWidths = @getScopedCharWidths(token.scopes)
valueIndex = 0
while valueIndex < value.length
if iterator.isPairedCharacter()
char = value
while valueIndex < token.value.length
if token.hasPairedCharacter
char = token.value.substr(valueIndex, 2)
charLength = 2
valueIndex += 2
else
char = value[valueIndex]
char = token.value[valueIndex]
charLength = 1
valueIndex++

5
src/language-mode.coffee
View File

@@ -242,9 +242,8 @@ class LanguageMode
@suggestedIndentForTokenizedLineAtBufferRow(bufferRow, tokenizedLine, options)
suggestedIndentForTokenizedLineAtBufferRow: (bufferRow, tokenizedLine, options) ->
iterator = tokenizedLine.getTokenIterator()
iterator.next()
scopeDescriptor = new ScopeDescriptor(scopes: iterator.getScopes())
scopes = tokenizedLine.tokens[0].scopes
scopeDescriptor = new ScopeDescriptor({scopes})
currentIndentLevel = @editor.indentationForBufferRow(bufferRow)
return currentIndentLevel unless increaseIndentRegex = @increaseIndentRegexForScopeDescriptor(scopeDescriptor)

160
src/lines-component.coffee
View File

@@ -4,13 +4,10 @@ _ = require 'underscore-plus'
CursorsComponent = require './cursors-component'
HighlightsComponent = require './highlights-component'
TokenIterator = require './token-iterator'
DummyLineNode = $$(-> @div className: 'line', style: 'position: absolute; visibility: hidden;', => @span 'x')[0]
AcceptFilter = {acceptNode: -> NodeFilter.FILTER_ACCEPT}
WrapperDiv = document.createElement('div')
TokenTextEscapeRegex = /[&"'<>]/g
MaxTokenLength = 20000
cloneObject = (object) ->
clone = {}
@@ -22,7 +19,6 @@ class LinesComponent
placeholderTextDiv: null
constructor: ({@presenter, @hostElement, @useShadowDOM, visible}) ->
@tokenIterator = new TokenIterator
@measuredLines = new Set
@lineNodesByLineId = {}
@screenRowsByLineId = {}
@@ -171,116 +167,20 @@ class LinesComponent
@buildEndOfLineHTML(id) or '&nbsp;'
buildLineInnerHTML: (id) ->
lineState = @newState.lines[id]
{firstNonWhitespaceIndex, firstTrailingWhitespaceIndex, invisibles} = lineState
lineIsWhitespaceOnly = firstTrailingWhitespaceIndex is 0
{indentGuidesVisible} = @newState
{tokens, text, isOnlyWhitespace} = @newState.lines[id]
innerHTML = ""
@tokenIterator.reset(lineState)
while @tokenIterator.next()
for scope in @tokenIterator.getScopeEnds()
innerHTML += "</span>"
for scope in @tokenIterator.getScopeStarts()
innerHTML += "<span class=\"#{scope.replace(/\.+/g, ' ')}\">"
tokenStart = @tokenIterator.getScreenStart()
tokenEnd = @tokenIterator.getScreenEnd()
tokenText = @tokenIterator.getText()
isHardTab = @tokenIterator.isHardTab()
if hasLeadingWhitespace = tokenStart < firstNonWhitespaceIndex
tokenFirstNonWhitespaceIndex = firstNonWhitespaceIndex - tokenStart
else
tokenFirstNonWhitespaceIndex = null
if hasTrailingWhitespace = tokenEnd > firstTrailingWhitespaceIndex
tokenFirstTrailingWhitespaceIndex = Math.max(0, firstTrailingWhitespaceIndex - tokenStart)
else
tokenFirstTrailingWhitespaceIndex = null
hasIndentGuide =
@newState.indentGuidesVisible and
(hasLeadingWhitespace or lineIsWhitespaceOnly)
hasInvisibleCharacters =
(invisibles?.tab and isHardTab) or
(invisibles?.space and (hasLeadingWhitespace or hasTrailingWhitespace))
innerHTML += @buildTokenHTML(tokenText, isHardTab, tokenFirstNonWhitespaceIndex, tokenFirstTrailingWhitespaceIndex, hasIndentGuide, hasInvisibleCharacters)
for scope in @tokenIterator.getScopeEnds()
innerHTML += "</span>"
for scope in @tokenIterator.getScopes()
innerHTML += "</span>"
scopeStack = []
for token in tokens
innerHTML += @updateScopeStack(scopeStack, token.scopes)
hasIndentGuide = indentGuidesVisible and (token.hasLeadingWhitespace() or (token.hasTrailingWhitespace() and isOnlyWhitespace))
innerHTML += token.getValueAsHtml({hasIndentGuide})
innerHTML += @popScope(scopeStack) while scopeStack.length > 0
innerHTML += @buildEndOfLineHTML(id)
innerHTML
buildTokenHTML: (tokenText, isHardTab, firstNonWhitespaceIndex, firstTrailingWhitespaceIndex, hasIndentGuide, hasInvisibleCharacters) ->
if isHardTab
classes = 'hard-tab'
classes += ' leading-whitespace' if firstNonWhitespaceIndex?
classes += ' trailing-whitespace' if firstTrailingWhitespaceIndex?
classes += ' indent-guide' if hasIndentGuide
classes += ' invisible-character' if hasInvisibleCharacters
return "<span class='#{classes}'>#{@escapeTokenText(tokenText)}</span>"
else
startIndex = 0
endIndex = tokenText.length
leadingHtml = ''
trailingHtml = ''
if firstNonWhitespaceIndex?
leadingWhitespace = tokenText.substring(0, firstNonWhitespaceIndex)
classes = 'leading-whitespace'
classes += ' indent-guide' if hasIndentGuide
classes += ' invisible-character' if hasInvisibleCharacters
leadingHtml = "<span class='#{classes}'>#{leadingWhitespace}</span>"
startIndex = firstNonWhitespaceIndex
if firstTrailingWhitespaceIndex?
tokenIsOnlyWhitespace = firstTrailingWhitespaceIndex is 0
trailingWhitespace = tokenText.substring(firstTrailingWhitespaceIndex)
classes = 'trailing-whitespace'
classes += ' indent-guide' if hasIndentGuide and not firstNonWhitespaceIndex? and tokenIsOnlyWhitespace
classes += ' invisible-character' if hasInvisibleCharacters
trailingHtml = "<span class='#{classes}'>#{trailingWhitespace}</span>"
endIndex = firstTrailingWhitespaceIndex
html = leadingHtml
if tokenText.length > MaxTokenLength
while startIndex < endIndex
html += "<span>" + @escapeTokenText(tokenText, startIndex, startIndex + MaxTokenLength) + "</span>"
startIndex += MaxTokenLength
else
html += @escapeTokenText(tokenText, startIndex, endIndex)
html += trailingHtml
html
escapeTokenText: (tokenText, startIndex, endIndex) ->
if startIndex? and endIndex? and startIndex > 0 or endIndex < tokenText.length
tokenText = tokenText.slice(startIndex, endIndex)
tokenText.replace(TokenTextEscapeRegex, @escapeTokenTextReplace)
escapeTokenTextReplace: (match) ->
switch match
when '&' then '&amp;'
when '"' then '&quot;'
when "'" then '&#39;'
when '<' then '&lt;'
when '>' then '&gt;'
else match
buildEndOfLineHTML: (id) ->
{endOfLineInvisibles} = @newState.lines[id]
@@ -290,6 +190,31 @@ class LinesComponent
html += "<span class='invisible-character'>#{invisible}</span>"
html
updateScopeStack: (scopeStack, desiredScopeDescriptor) ->
html = ""
# Find a common prefix
for scope, i in desiredScopeDescriptor
break unless scopeStack[i] is desiredScopeDescriptor[i]
# Pop scopeDescriptor until we're at the common prefx
until scopeStack.length is i
html += @popScope(scopeStack)
# Push onto common prefix until scopeStack equals desiredScopeDescriptor
for j in [i...desiredScopeDescriptor.length]
html += @pushScope(scopeStack, desiredScopeDescriptor[j])
html
popScope: (scopeStack) ->
scopeStack.pop()
"</span>"
pushScope: (scopeStack, scope) ->
scopeStack.push(scope)
"<span class=\"#{scope.replace(/\.+/g, ' ')}\">"
updateLineNode: (id) ->
oldLineState = @oldState.lines[id]
newLineState = @newState.lines[id]
@@ -354,22 +279,19 @@ class LinesComponent
iterator = null
charIndex = 0
@tokenIterator.reset(tokenizedLine)
while @tokenIterator.next()
scopes = @tokenIterator.getScopes()
text = @tokenIterator.getText()
for {value, scopes, hasPairedCharacter} in tokenizedLine.tokens
charWidths = @presenter.getScopedCharacterWidths(scopes)
textIndex = 0
while textIndex < text.length
if @tokenIterator.isPairedCharacter()
char = text
valueIndex = 0
while valueIndex < value.length
if hasPairedCharacter
char = value.substr(valueIndex, 2)
charLength = 2
textIndex += 2
valueIndex += 2
else
char = text[textIndex]
char = value[valueIndex]
charLength = 1
textIndex++
valueIndex++
continue if char is '\0'

6
src/special-token-symbols.coffee
View File

@@ -1,6 +0,0 @@
module.exports = {
SoftTab: Symbol('SoftTab')
HardTab: Symbol('HardTab')
PairedCharacter: Symbol('PairedCharacter')
SoftWrapIndent: Symbol('SoftWrapIndent')
}

24
src/text-editor-presenter.coffee
View File

@@ -336,14 +336,9 @@ class TextEditorPresenter
@state.content.lines[line.id] =
screenRow: row
text: line.text
openScopes: line.openScopes
tags: line.tags
specialTokens: line.specialTokens
firstNonWhitespaceIndex: line.firstNonWhitespaceIndex
firstTrailingWhitespaceIndex: line.firstTrailingWhitespaceIndex
invisibles: line.invisibles
endOfLineInvisibles: line.endOfLineInvisibles
tokens: line.tokens
isOnlyWhitespace: line.isOnlyWhitespace()
endOfLineInvisibles: line.endOfLineInvisibles
indentLevel: line.indentLevel
tabLength: line.tabLength
fold: line.fold
@@ -1011,20 +1006,17 @@ class TextEditorPresenter
top = targetRow * @lineHeight
left = 0
column = 0
iterator = @model.tokenizedLineForScreenRow(targetRow).getTokenIterator()
while iterator.next()
characterWidths = @getScopedCharacterWidths(iterator.getScopes())
for token in @model.tokenizedLineForScreenRow(targetRow).tokens
characterWidths = @getScopedCharacterWidths(token.scopes)
valueIndex = 0
text = iterator.getText()
while valueIndex < text.length
if iterator.isPairedCharacter()
char = text
while valueIndex < token.value.length
if token.hasPairedCharacter
char = token.value.substr(valueIndex, 2)
charLength = 2
valueIndex += 2
else
char = text[valueIndex]
char = token.value[valueIndex]
charLength = 1
valueIndex++

3
src/text-editor.coffee
View File

@@ -2457,8 +2457,9 @@ class TextEditor extends Model
# Extended: Determine if the given row is entirely a comment
isBufferRowCommented: (bufferRow) ->
if match = @lineTextForBufferRow(bufferRow).match(/\S/)
scopeDescriptor = @tokenForBufferPosition([bufferRow, match.index]).scopes
@commentScopeSelector ?= new TextMateScopeSelector('comment.*')
@commentScopeSelector.matches(@scopeDescriptorForBufferPosition([bufferRow, match.index]).scopes)
@commentScopeSelector.matches(scopeDescriptor)
logCursorScope: ->
scopeDescriptor = @getLastCursor().getScopeDescriptor()

83
src/token-iterator.coffee
View File

@@ -1,83 +0,0 @@
{SoftTab, HardTab, PairedCharacter, SoftWrapIndent} = require './special-token-symbols'
module.exports =
class TokenIterator
constructor: (line) ->
@reset(line) if line?
reset: (@line) ->
@index = null
@bufferStart = @line.startBufferColumn
@bufferEnd = @bufferStart
@screenStart = 0
@screenEnd = 0
@scopes = @line.openScopes.map (id) -> atom.grammars.scopeForId(id)
@scopeStarts = @scopes.slice()
@scopeEnds = []
this
next: ->
{tags} = @line
if @index?
@index++
@scopeEnds.length = 0
@scopeStarts.length = 0
@bufferStart = @bufferEnd
@screenStart = @screenEnd
else
@index = 0
while @index < tags.length
tag = tags[@index]
if tag < 0
if tag % 2 is 0
@scopeEnds.push(atom.grammars.scopeForId(tag + 1))
@scopes.pop()
else
scope = atom.grammars.scopeForId(tag)
@scopeStarts.push(scope)
@scopes.push(scope)
@index++
else
if @isHardTab()
@screenEnd = @screenStart + tag
@bufferEnd = @bufferStart + 1
else if @isSoftWrapIndentation()
@screenEnd = @screenStart + tag
@bufferEnd = @bufferStart + 0
else
@screenEnd = @screenStart + tag
@bufferEnd = @bufferStart + tag
return true
false
getBufferStart: -> @bufferStart
getBufferEnd: -> @bufferEnd
getScreenStart: -> @screenStart
getScreenEnd: -> @screenEnd
getScopeStarts: -> @scopeStarts
getScopeEnds: -> @scopeEnds
getScopes: -> @scopes
getText: ->
@line.text.substring(@screenStart, @screenEnd)
isSoftTab: ->
@line.specialTokens[@index] is SoftTab
isHardTab: ->
@line.specialTokens[@index] is HardTab
isSoftWrapIndentation: ->
@line.specialTokens[@index] is SoftWrapIndent
isPairedCharacter: ->
@line.specialTokens[@index] is PairedCharacter
isAtomic: ->
@isSoftTab() or @isHardTab() or @isSoftWrapIndentation() or @isPairedCharacter()

199
src/token.coffee
View File

@@ -1,8 +1,13 @@
_ = require 'underscore-plus'
textUtils = require './text-utils'
WhitespaceRegexesByTabLength = {}
EscapeRegex = /[&"'<>]/g
StartDotRegex = /^\.?/
WhitespaceRegex = /\S/
MaxTokenLength = 20000
# Represents a single unit of text as selected by a grammar.
module.exports =
class Token
@@ -15,14 +20,10 @@ class Token
firstTrailingWhitespaceIndex: null
hasInvisibleCharacters: false
constructor: (properties) ->
{@value, @scopes, @isAtomic, @isHardTab, @bufferDelta} = properties
{@hasInvisibleCharacters, @hasPairedCharacter, @isSoftWrapIndentation} = properties
@firstNonWhitespaceIndex = properties.firstNonWhitespaceIndex ? null
@firstTrailingWhitespaceIndex = properties.firstTrailingWhitespaceIndex ? null
constructor: ({@value, @scopes, @isAtomic, @bufferDelta, @isHardTab, @hasPairedCharacter, @isSoftWrapIndentation}) ->
@screenDelta = @value.length
@bufferDelta ?= @screenDelta
@hasPairedCharacter ?= textUtils.hasPairedCharacter(@value)
isEqual: (other) ->
# TODO: scopes is deprecated. This is here for the sake of lang package tests
@@ -31,6 +32,126 @@ class Token
isBracket: ->
/^meta\.brace\b/.test(_.last(@scopes))
splitAt: (splitIndex) ->
leftToken = new Token(value: @value.substring(0, splitIndex), scopes: @scopes)
rightToken = new Token(value: @value.substring(splitIndex), scopes: @scopes)
if @firstNonWhitespaceIndex?
leftToken.firstNonWhitespaceIndex = Math.min(splitIndex, @firstNonWhitespaceIndex)
leftToken.hasInvisibleCharacters = @hasInvisibleCharacters
if @firstNonWhitespaceIndex > splitIndex
rightToken.firstNonWhitespaceIndex = @firstNonWhitespaceIndex - splitIndex
rightToken.hasInvisibleCharacters = @hasInvisibleCharacters
if @firstTrailingWhitespaceIndex?
rightToken.firstTrailingWhitespaceIndex = Math.max(0, @firstTrailingWhitespaceIndex - splitIndex)
rightToken.hasInvisibleCharacters = @hasInvisibleCharacters
if @firstTrailingWhitespaceIndex < splitIndex
leftToken.firstTrailingWhitespaceIndex = @firstTrailingWhitespaceIndex
leftToken.hasInvisibleCharacters = @hasInvisibleCharacters
[leftToken, rightToken]
whitespaceRegexForTabLength: (tabLength) ->
WhitespaceRegexesByTabLength[tabLength] ?= new RegExp("([ ]{#{tabLength}})|(\t)|([^\t]+)", "g")
breakOutAtomicTokens: (tabLength, breakOutLeadingSoftTabs, startColumn) ->
if @hasPairedCharacter
outputTokens = []
column = startColumn
for token in @breakOutPairedCharacters()
if token.isAtomic
outputTokens.push(token)
else
outputTokens.push(token.breakOutAtomicTokens(tabLength, breakOutLeadingSoftTabs, column)...)
breakOutLeadingSoftTabs = token.isOnlyWhitespace() if breakOutLeadingSoftTabs
column += token.value.length
outputTokens
else
return [this] if @isAtomic
if breakOutLeadingSoftTabs
return [this] unless /^[ ]|\t/.test(@value)
else
return [this] unless /\t/.test(@value)
outputTokens = []
regex = @whitespaceRegexForTabLength(tabLength)
column = startColumn
while match = regex.exec(@value)
[fullMatch, softTab, hardTab] = match
token = null
if softTab and breakOutLeadingSoftTabs
token = @buildSoftTabToken(tabLength)
else if hardTab
breakOutLeadingSoftTabs = false
token = @buildHardTabToken(tabLength, column)
else
breakOutLeadingSoftTabs = false
value = match[0]
token = new Token({value, @scopes})
column += token.value.length
outputTokens.push(token)
outputTokens
breakOutPairedCharacters: ->
outputTokens = []
index = 0
nonPairStart = 0
while index < @value.length
if textUtils.isPairedCharacter(@value, index)
if nonPairStart isnt index
outputTokens.push(new Token({value: @value[nonPairStart...index], @scopes}))
outputTokens.push(@buildPairedCharacterToken(@value, index))
index += 2
nonPairStart = index
else
index++
if nonPairStart isnt index
outputTokens.push(new Token({value: @value[nonPairStart...index], @scopes}))
outputTokens
buildPairedCharacterToken: (value, index) ->
new Token(
value: value[index..index + 1]
scopes: @scopes
isAtomic: true
hasPairedCharacter: true
)
buildHardTabToken: (tabLength, column) ->
@buildTabToken(tabLength, true, column)
buildSoftTabToken: (tabLength) ->
@buildTabToken(tabLength, false, 0)
buildTabToken: (tabLength, isHardTab, column=0) ->
tabStop = tabLength - (column % tabLength)
new Token(
value: _.multiplyString(" ", tabStop)
scopes: @scopes
bufferDelta: if isHardTab then 1 else tabStop
isAtomic: true
isHardTab: isHardTab
)
buildSoftWrapIndentationToken: (length) ->
new Token(
value: _.multiplyString(" ", length),
scopes: @scopes,
bufferDelta: 0,
isAtomic: true,
isSoftWrapIndentation: true
)
isOnlyWhitespace: ->
not WhitespaceRegex.test(@value)
@@ -40,6 +161,72 @@ class Token
scopeClasses = scope.split('.')
_.isSubset(targetClasses, scopeClasses)
getValueAsHtml: ({hasIndentGuide}) ->
if @isHardTab
classes = 'hard-tab'
classes += ' leading-whitespace' if @hasLeadingWhitespace()
classes += ' trailing-whitespace' if @hasTrailingWhitespace()
classes += ' indent-guide' if hasIndentGuide
classes += ' invisible-character' if @hasInvisibleCharacters
html = "<span class='#{classes}'>#{@escapeString(@value)}</span>"
else
startIndex = 0
endIndex = @value.length
leadingHtml = ''
trailingHtml = ''
if @hasLeadingWhitespace()
leadingWhitespace = @value.substring(0, @firstNonWhitespaceIndex)
classes = 'leading-whitespace'
classes += ' indent-guide' if hasIndentGuide
classes += ' invisible-character' if @hasInvisibleCharacters
leadingHtml = "<span class='#{classes}'>#{leadingWhitespace}</span>"
startIndex = @firstNonWhitespaceIndex
if @hasTrailingWhitespace()
tokenIsOnlyWhitespace = @firstTrailingWhitespaceIndex is 0
trailingWhitespace = @value.substring(@firstTrailingWhitespaceIndex)
classes = 'trailing-whitespace'
classes += ' indent-guide' if hasIndentGuide and not @hasLeadingWhitespace() and tokenIsOnlyWhitespace
classes += ' invisible-character' if @hasInvisibleCharacters
trailingHtml = "<span class='#{classes}'>#{trailingWhitespace}</span>"
endIndex = @firstTrailingWhitespaceIndex
html = leadingHtml
if @value.length > MaxTokenLength
while startIndex < endIndex
html += "<span>" + @escapeString(@value, startIndex, startIndex + MaxTokenLength) + "</span>"
startIndex += MaxTokenLength
else
html += @escapeString(@value, startIndex, endIndex)
html += trailingHtml
html
escapeString: (str, startIndex, endIndex) ->
strLength = str.length
startIndex ?= 0
endIndex ?= strLength
str = str.slice(startIndex, endIndex) if startIndex > 0 or endIndex < strLength
str.replace(EscapeRegex, @escapeStringReplace)
escapeStringReplace: (match) ->
switch match
when '&' then '&amp;'
when '"' then '&quot;'
when "'" then '&#39;'
when '<' then '&lt;'
when '>' then '&gt;'
else match
hasLeadingWhitespace: ->
@firstNonWhitespaceIndex? and @firstNonWhitespaceIndex > 0

184
src/tokenized-buffer.coffee
View File

@@ -1,11 +1,9 @@
_ = require 'underscore-plus'
{CompositeDisposable, Emitter} = require 'event-kit'
{Point, Range} = require 'text-buffer'
{ScopeSelector} = require 'first-mate'
Serializable = require 'serializable'
Model = require './model'
TokenizedLine = require './tokenized-line'
TokenIterator = require './token-iterator'
Token = require './token'
ScopeDescriptor = require './scope-descriptor'
Grim = require 'grim'
@@ -27,7 +25,6 @@ class TokenizedBuffer extends Model
constructor: ({@buffer, @tabLength, @ignoreInvisibles}) ->
@emitter = new Emitter
@disposables = new CompositeDisposable
@tokenIterator = new TokenIterator
@disposables.add atom.grammars.onDidAddGrammar(@grammarAddedOrUpdated)
@disposables.add atom.grammars.onDidUpdateGrammar(@grammarAddedOrUpdated)
@@ -170,7 +167,7 @@ class TokenizedBuffer extends Model
row = startRow
loop
previousStack = @stackForRow(row)
@tokenizedLines[row] = @buildTokenizedLineForRow(row, @stackForRow(row - 1), @openScopesForRow(row))
@tokenizedLines[row] = @buildTokenizedLineForRow(row, @stackForRow(row - 1))
if --rowsRemaining is 0
filledRegion = false
endRow = row
@@ -230,7 +227,7 @@ class TokenizedBuffer extends Model
@updateInvalidRows(start, end, delta)
previousEndStack = @stackForRow(end) # used in spill detection below
newTokenizedLines = @buildTokenizedLinesForRows(start, end + delta, @stackForRow(start - 1), @openScopesForRow(start))
newTokenizedLines = @buildTokenizedLinesForRows(start, end + delta, @stackForRow(start - 1))
_.spliceWithArray(@tokenizedLines, start, end - start + 1, newTokenizedLines)
start = @retokenizeWhitespaceRowsIfIndentLevelChanged(start - 1, -1)
@@ -251,7 +248,7 @@ class TokenizedBuffer extends Model
line = @tokenizedLines[row]
if line?.isOnlyWhitespace() and @indentLevelForRow(row) isnt line.indentLevel
while line?.isOnlyWhitespace()
@tokenizedLines[row] = @buildTokenizedLineForRow(row, @stackForRow(row - 1), @openScopesForRow(row))
@tokenizedLines[row] = @buildTokenizedLineForRow(row, @stackForRow(row - 1))
row += increment
line = @tokenizedLines[row]
@@ -293,18 +290,16 @@ class TokenizedBuffer extends Model
@tokenizedLineForRow(row).isComment() and
@tokenizedLineForRow(nextRow).isComment()
buildTokenizedLinesForRows: (startRow, endRow, startingStack, startingopenScopes) ->
buildTokenizedLinesForRows: (startRow, endRow, startingStack) ->
ruleStack = startingStack
openScopes = startingopenScopes
stopTokenizingAt = startRow + @chunkSize
tokenizedLines = for row in [startRow..endRow]
if (ruleStack or row is 0) and row < stopTokenizingAt
tokenizedLine = @buildTokenizedLineForRow(row, ruleStack, openScopes)
ruleStack = tokenizedLine.ruleStack
openScopes = @scopesFromTags(openScopes, tokenizedLine.tags)
screenLine = @buildTokenizedLineForRow(row, ruleStack)
ruleStack = screenLine.ruleStack
else
tokenizedLine = @buildPlaceholderTokenizedLineForRow(row, openScopes)
tokenizedLine
screenLine = @buildPlaceholderTokenizedLineForRow(row)
screenLine
if endRow >= stopTokenizingAt
@invalidateRow(stopTokenizingAt)
@@ -316,23 +311,22 @@ class TokenizedBuffer extends Model
@buildPlaceholderTokenizedLineForRow(row) for row in [startRow..endRow]
buildPlaceholderTokenizedLineForRow: (row) ->
openScopes = [@grammar.startIdForScope(@grammar.scopeName)]
text = @buffer.lineForRow(row)
tags = [text.length]
line = @buffer.lineForRow(row)
tokens = [new Token(value: line, scopes: [@grammar.scopeName])]
tabLength = @getTabLength()
indentLevel = @indentLevelForRow(row)
lineEnding = @buffer.lineEndingForRow(row)
new TokenizedLine({openScopes, text, tags, tabLength, indentLevel, invisibles: @getInvisiblesToShow(), lineEnding, @tokenIterator})
new TokenizedLine({tokens, tabLength, indentLevel, invisibles: @getInvisiblesToShow(), lineEnding})
buildTokenizedLineForRow: (row, ruleStack, openScopes) ->
@buildTokenizedLineForRowWithText(row, @buffer.lineForRow(row), ruleStack, openScopes)
buildTokenizedLineForRow: (row, ruleStack) ->
@buildTokenizedLineForRowWithText(row, @buffer.lineForRow(row), ruleStack)
buildTokenizedLineForRowWithText: (row, text, ruleStack = @stackForRow(row - 1), openScopes = @openScopesForRow(row)) ->
buildTokenizedLineForRowWithText: (row, line, ruleStack = @stackForRow(row - 1)) ->
lineEnding = @buffer.lineEndingForRow(row)
tabLength = @getTabLength()
indentLevel = @indentLevelForRow(row)
{tags, ruleStack} = @grammar.tokenizeLine(text, ruleStack, row is 0, false)
new TokenizedLine({openScopes, text, tags, ruleStack, tabLength, lineEnding, indentLevel, invisibles: @getInvisiblesToShow(), @tokenIterator})
{tokens, ruleStack} = @grammar.tokenizeLine(line, ruleStack, row is 0)
new TokenizedLine({tokens, ruleStack, tabLength, lineEnding, indentLevel, invisibles: @getInvisiblesToShow()})
getInvisiblesToShow: ->
if @configSettings.showInvisibles and not @ignoreInvisibles
@@ -346,25 +340,6 @@ class TokenizedBuffer extends Model
stackForRow: (bufferRow) ->
@tokenizedLines[bufferRow]?.ruleStack
openScopesForRow: (bufferRow) ->
if bufferRow > 0
precedingLine = @tokenizedLines[bufferRow - 1]
@scopesFromTags(precedingLine.openScopes, precedingLine.tags)
else
[]
scopesFromTags: (startingScopes, tags) ->
scopes = startingScopes.slice()
for tag in tags when tag < 0
if (tag % 2) is -1
scopes.push(tag)
else
expectedScope = tag + 1
poppedScope = scopes.pop()
unless poppedScope is expectedScope
throw new Error("Encountered an invalid scope end id. Popped #{poppedScope}, expected to pop #{expectedScope}.")
scopes
indentLevelForRow: (bufferRow) ->
line = @buffer.lineForRow(bufferRow)
indentLevel = 0
@@ -401,20 +376,7 @@ class TokenizedBuffer extends Model
0
scopeDescriptorForPosition: (position) ->
{row, column} = Point.fromObject(position)
iterator = @tokenizedLines[row].getTokenIterator()
while iterator.next()
if iterator.getScreenEnd() > column
scopes = iterator.getScopes()
break
# rebuild scope of last token if we iterated off the end
unless scopes?
scopes = iterator.getScopes()
scopes.push(iterator.getScopeEnds().reverse()...)
new ScopeDescriptor({scopes})
new ScopeDescriptor(scopes: @tokenForPosition(position).scopes)
tokenForPosition: (position) ->
{row, column} = Point.fromObject(position)
@@ -426,53 +388,85 @@ class TokenizedBuffer extends Model
new Point(row, column)
bufferRangeForScopeAtPosition: (selector, position) ->
selector = new ScopeSelector(selector.replace(/^\./, ''))
position = Point.fromObject(position)
tokenizedLine = @tokenizedLines[position.row]
startIndex = tokenizedLine.tokenIndexAtBufferColumn(position.column)
{openScopes, tags} = @tokenizedLines[position.row]
scopes = openScopes.map (tag) -> atom.grammars.scopeForId(tag)
for index in [startIndex..0]
token = tokenizedLine.tokenAtIndex(index)
break unless token.matchesScopeSelector(selector)
firstToken = token
startColumn = 0
for tag, tokenIndex in tags
if tag < 0
if tag % 2 is -1
scopes.push(atom.grammars.scopeForId(tag))
else
scopes.pop()
else
endColumn = startColumn + tag
if endColumn > position.column
break
else
startColumn = endColumn
for index in [startIndex...tokenizedLine.getTokenCount()]
token = tokenizedLine.tokenAtIndex(index)
break unless token.matchesScopeSelector(selector)
lastToken = token
return unless selector.matches(scopes)
return unless firstToken? and lastToken?
startScopes = scopes.slice()
for startTokenIndex in [(tokenIndex - 1)..0] by -1
tag = tags[startTokenIndex]
if tag < 0
if tag % 2 is -1
startScopes.pop()
else
startScopes.push(atom.grammars.scopeForId(tag))
else
break unless selector.matches(startScopes)
startColumn -= tag
startColumn = tokenizedLine.bufferColumnForToken(firstToken)
endColumn = tokenizedLine.bufferColumnForToken(lastToken) + lastToken.bufferDelta
new Range([position.row, startColumn], [position.row, endColumn])
endScopes = scopes.slice()
for endTokenIndex in [(tokenIndex + 1)...tags.length] by 1
tag = tags[endTokenIndex]
if tag < 0
if tag % 2 is -1
endScopes.push(atom.grammars.scopeForId(tag))
else
endScopes.pop()
else
break unless selector.matches(endScopes)
endColumn += tag
iterateTokensInBufferRange: (bufferRange, iterator) ->
bufferRange = Range.fromObject(bufferRange)
{start, end} = bufferRange
new Range(new Point(position.row, startColumn), new Point(position.row, endColumn))
keepLooping = true
stop = -> keepLooping = false
for bufferRow in [start.row..end.row]
bufferColumn = 0
for token in @tokenizedLines[bufferRow].tokens
startOfToken = new Point(bufferRow, bufferColumn)
iterator(token, startOfToken, {stop}) if bufferRange.containsPoint(startOfToken)
return unless keepLooping
bufferColumn += token.bufferDelta
backwardsIterateTokensInBufferRange: (bufferRange, iterator) ->
bufferRange = Range.fromObject(bufferRange)
{start, end} = bufferRange
keepLooping = true
stop = -> keepLooping = false
for bufferRow in [end.row..start.row]
bufferColumn = @buffer.lineLengthForRow(bufferRow)
for token in new Array(@tokenizedLines[bufferRow].tokens...).reverse()
bufferColumn -= token.bufferDelta
startOfToken = new Point(bufferRow, bufferColumn)
iterator(token, startOfToken, {stop}) if bufferRange.containsPoint(startOfToken)
return unless keepLooping
findOpeningBracket: (startBufferPosition) ->
range = [[0,0], startBufferPosition]
position = null
depth = 0
@backwardsIterateTokensInBufferRange range, (token, startPosition, {stop}) ->
if token.isBracket()
if token.value is '}'
depth++
else if token.value is '{'
depth--
if depth is 0
position = startPosition
stop()
position
findClosingBracket: (startBufferPosition) ->
range = [startBufferPosition, @buffer.getEndPosition()]
position = null
depth = 0
@iterateTokensInBufferRange range, (token, startPosition, {stop}) ->
if token.isBracket()
if token.value is '{'
depth++
else if token.value is '}'
depth--
if depth is 0
position = startPosition
stop()
position
# Gets the row number of the last line.
#

524
src/tokenized-line.coffee
View File

@@ -1,13 +1,10 @@
_ = require 'underscore-plus'
{isPairedCharacter} = require './text-utils'
Token = require './token'
{SoftTab, HardTab, PairedCharacter, SoftWrapIndent} = require './special-token-symbols'
NonWhitespaceRegex = /\S/
LeadingWhitespaceRegex = /^\s*/
TrailingWhitespaceRegex = /\s*$/
RepeatedSpaceRegex = /[ ]/g
CommentScopeRegex = /(\b|\.)comment/
idCounter = 1
module.exports =
@@ -17,181 +14,32 @@ class TokenizedLine
firstNonWhitespaceIndex: 0
foldable: false
constructor: (properties) ->
@id = idCounter++
return unless properties?
@specialTokens = {}
{@openScopes, @text, @tags, @lineEnding, @ruleStack, @tokenIterator} = properties
{@startBufferColumn, @fold, @tabLength, @indentLevel, @invisibles} = properties
constructor: ({tokens, @lineEnding, @ruleStack, @startBufferColumn, @fold, @tabLength, @indentLevel, @invisibles}) ->
@startBufferColumn ?= 0
@bufferDelta = @text.length
@tokens = @breakOutAtomicTokens(tokens)
@text = @buildText()
@bufferDelta = @buildBufferDelta()
@softWrapIndentationTokens = @getSoftWrapIndentationTokens()
@softWrapIndentationDelta = @buildSoftWrapIndentationDelta()
@transformContent()
@buildEndOfLineInvisibles() if @invisibles? and @lineEnding?
@id = idCounter++
@markLeadingAndTrailingWhitespaceTokens()
if @invisibles
@substituteInvisibleCharacters()
@buildEndOfLineInvisibles() if @lineEnding?
transformContent: ->
text = ''
bufferColumn = 0
screenColumn = 0
tokenIndex = 0
tokenOffset = 0
firstNonWhitespaceColumn = null
lastNonWhitespaceColumn = null
buildText: ->
text = ""
text += token.value for token in @tokens
text
while bufferColumn < @text.length
# advance to next token if we've iterated over its length
if tokenOffset is @tags[tokenIndex]
tokenIndex++
tokenOffset = 0
# advance to next token tag
tokenIndex++ while @tags[tokenIndex] < 0
character = @text[bufferColumn]
# split out unicode surrogate pairs
if isPairedCharacter(@text, bufferColumn)
prefix = tokenOffset
suffix = @tags[tokenIndex] - tokenOffset - 2
splitTokens = []
splitTokens.push(prefix) if prefix > 0
splitTokens.push(2)
splitTokens.push(suffix) if suffix > 0
@tags.splice(tokenIndex, 1, splitTokens...)
firstNonWhitespaceColumn ?= screenColumn
lastNonWhitespaceColumn = screenColumn + 1
text += @text.substr(bufferColumn, 2)
screenColumn += 2
bufferColumn += 2
tokenIndex++ if prefix > 0
@specialTokens[tokenIndex] = PairedCharacter
tokenIndex++
tokenOffset = 0
# split out leading soft tabs
else if character is ' '
if firstNonWhitespaceColumn?
text += ' '
else
if (screenColumn + 1) % @tabLength is 0
@specialTokens[tokenIndex] = SoftTab
suffix = @tags[tokenIndex] - @tabLength
@tags.splice(tokenIndex, 1, @tabLength)
@tags.splice(tokenIndex + 1, 0, suffix) if suffix > 0
text += @invisibles?.space ? ' '
screenColumn++
bufferColumn++
tokenOffset++
# expand hard tabs to the next tab stop
else if character is '\t'
tabLength = @tabLength - (screenColumn % @tabLength)
if @invisibles?.tab
text += @invisibles.tab
else
text += ' '
text += ' ' for i in [1...tabLength] by 1
prefix = tokenOffset
suffix = @tags[tokenIndex] - tokenOffset - 1
splitTokens = []
splitTokens.push(prefix) if prefix > 0
splitTokens.push(tabLength)
splitTokens.push(suffix) if suffix > 0
@tags.splice(tokenIndex, 1, splitTokens...)
screenColumn += tabLength
bufferColumn++
tokenIndex++ if prefix > 0
@specialTokens[tokenIndex] = HardTab
tokenIndex++
tokenOffset = 0
# continue past any other character
else
firstNonWhitespaceColumn ?= screenColumn
lastNonWhitespaceColumn = screenColumn
text += character
screenColumn++
bufferColumn++
tokenOffset++
@text = text
@firstNonWhitespaceIndex = firstNonWhitespaceColumn
if lastNonWhitespaceColumn?
if lastNonWhitespaceColumn + 1 < @text.length
@firstTrailingWhitespaceIndex = lastNonWhitespaceColumn + 1
if @invisibles?.space
@text =
@text.substring(0, @firstTrailingWhitespaceIndex) +
@text.substring(@firstTrailingWhitespaceIndex)
.replace(RepeatedSpaceRegex, @invisibles.space)
else
@lineIsWhitespaceOnly = true
@firstTrailingWhitespaceIndex = 0
getTokenIterator: -> @tokenIterator.reset(this)
Object.defineProperty @prototype, 'tokens', get: ->
iterator = @getTokenIterator()
tokens = []
while iterator.next()
properties = {
value: iterator.getText()
scopes: iterator.getScopes().slice()
isAtomic: iterator.isAtomic()
isHardTab: iterator.isHardTab()
hasPairedCharacter: iterator.isPairedCharacter()
isSoftWrapIndentation: iterator.isSoftWrapIndentation()
}
if iterator.isHardTab()
properties.bufferDelta = 1
properties.hasInvisibleCharacters = true if @invisibles?.tab
if iterator.getScreenStart() < @firstNonWhitespaceIndex
properties.firstNonWhitespaceIndex =
Math.min(@firstNonWhitespaceIndex, iterator.getScreenEnd()) - iterator.getScreenStart()
properties.hasInvisibleCharacters = true if @invisibles?.space
if @lineEnding? and iterator.getScreenEnd() > @firstTrailingWhitespaceIndex
properties.firstTrailingWhitespaceIndex =
Math.max(0, @firstTrailingWhitespaceIndex - iterator.getScreenStart())
properties.hasInvisibleCharacters = true if @invisibles?.space
tokens.push(new Token(properties))
tokens
buildBufferDelta: ->
delta = 0
delta += token.bufferDelta for token in @tokens
delta
copy: ->
copy = new TokenizedLine
copy.tokenIterator = @tokenIterator
copy.indentLevel = @indentLevel
copy.openScopes = @openScopes
copy.text = @text
copy.tags = @tags
copy.specialTokens = @specialTokens
copy.firstNonWhitespaceIndex = @firstNonWhitespaceIndex
copy.firstTrailingWhitespaceIndex = @firstTrailingWhitespaceIndex
copy.lineEnding = @lineEnding
copy.endOfLineInvisibles = @endOfLineInvisibles
copy.ruleStack = @ruleStack
copy.startBufferColumn = @startBufferColumn
copy.fold = @fold
copy
new TokenizedLine({@tokens, @lineEnding, @ruleStack, @startBufferColumn, @fold})
# This clips a given screen column to a valid column that's within the line
# and not in the middle of any atomic tokens.
@@ -204,58 +52,49 @@ class TokenizedLine
#
# Returns a {Number} representing the clipped column.
clipScreenColumn: (column, options={}) ->
return 0 if @tags.length is 0
return 0 if @tokens.length is 0
{clip} = options
column = Math.min(column, @getMaxScreenColumn())
tokenStartColumn = 0
for token in @tokens
break if tokenStartColumn + token.screenDelta > column
tokenStartColumn += token.screenDelta
iterator = @getTokenIterator()
while iterator.next()
break if iterator.getScreenEnd() > column
if iterator.isSoftWrapIndentation()
iterator.next() while iterator.isSoftWrapIndentation()
iterator.getScreenStart()
else if iterator.isAtomic() and iterator.getScreenStart() < column
if @isColumnInsideSoftWrapIndentation(tokenStartColumn)
@softWrapIndentationDelta
else if token.isAtomic and tokenStartColumn < column
if clip is 'forward'
iterator.getScreenEnd()
tokenStartColumn + token.screenDelta
else if clip is 'backward'
iterator.getScreenStart()
tokenStartColumn
else #'closest'
if column > ((iterator.getScreenStart() + iterator.getScreenEnd()) / 2)
iterator.getScreenEnd()
if column > tokenStartColumn + (token.screenDelta / 2)
tokenStartColumn + token.screenDelta
else
iterator.getScreenStart()
tokenStartColumn
else
column
screenColumnForBufferColumn: (targetBufferColumn, options) ->
iterator = @getTokenIterator()
while iterator.next()
tokenBufferStart = iterator.getBufferStart()
tokenBufferEnd = iterator.getBufferEnd()
if tokenBufferStart <= targetBufferColumn < tokenBufferEnd
overshoot = targetBufferColumn - tokenBufferStart
return Math.min(
iterator.getScreenStart() + overshoot,
iterator.getScreenEnd()
)
iterator.getScreenEnd()
screenColumnForBufferColumn: (bufferColumn, options) ->
bufferColumn = bufferColumn - @startBufferColumn
screenColumn = 0
currentBufferColumn = 0
for token in @tokens
break if currentBufferColumn + token.bufferDelta > bufferColumn
screenColumn += token.screenDelta
currentBufferColumn += token.bufferDelta
@clipScreenColumn(screenColumn + (bufferColumn - currentBufferColumn))
bufferColumnForScreenColumn: (targetScreenColumn) ->
iterator = @getTokenIterator()
while iterator.next()
tokenScreenStart = iterator.getScreenStart()
tokenScreenEnd = iterator.getScreenEnd()
if tokenScreenStart <= targetScreenColumn < tokenScreenEnd
overshoot = targetScreenColumn - tokenScreenStart
return Math.min(
iterator.getBufferStart() + overshoot,
iterator.getBufferEnd()
)
iterator.getBufferEnd()
bufferColumnForScreenColumn: (screenColumn, options) ->
bufferColumn = @startBufferColumn
currentScreenColumn = 0
for token in @tokens
break if currentScreenColumn + token.screenDelta > screenColumn
bufferColumn += token.bufferDelta
currentScreenColumn += token.screenDelta
bufferColumn + (screenColumn - currentScreenColumn)
getMaxScreenColumn: ->
if @fold
@@ -289,128 +128,69 @@ class TokenizedLine
return maxColumn
buildSoftWrapIndentationTokens: (token, hangingIndent) ->
totalIndentSpaces = (@indentLevel * @tabLength) + hangingIndent
indentTokens = []
while totalIndentSpaces > 0
tokenLength = Math.min(@tabLength, totalIndentSpaces)
indentToken = token.buildSoftWrapIndentationToken(tokenLength)
indentTokens.push(indentToken)
totalIndentSpaces -= tokenLength
indentTokens
softWrapAt: (column, hangingIndent) ->
return [null, this] if column is 0
return [new TokenizedLine([], '', [0, 0], [0, 0]), this] if column is 0
leftText = @text.substring(0, column)
rightText = @text.substring(column)
rightTokens = new Array(@tokens...)
leftTokens = []
leftScreenColumn = 0
leftTags = []
rightTags = []
while leftScreenColumn < column
if leftScreenColumn + rightTokens[0].screenDelta > column
rightTokens[0..0] = rightTokens[0].splitAt(column - leftScreenColumn)
nextToken = rightTokens.shift()
leftScreenColumn += nextToken.screenDelta
leftTokens.push nextToken
leftSpecialTokens = {}
rightSpecialTokens = {}
rightOpenScopes = @openScopes.slice()
screenColumn = 0
for tag, index in @tags
# tag represents a token
if tag >= 0
# token ends before the soft wrap column
if screenColumn + tag <= column
if specialToken = @specialTokens[index]
leftSpecialTokens[index] = specialToken
leftTags.push(tag)
screenColumn += tag
# token starts before and ends after the split column
else if screenColumn <= column
leftSuffix = column - screenColumn
rightPrefix = screenColumn + tag - column
leftTags.push(leftSuffix) if leftSuffix > 0
softWrapIndent = @indentLevel * @tabLength + (hangingIndent ? 0)
for i in [0...softWrapIndent] by 1
rightText = ' ' + rightText
remainingSoftWrapIndent = softWrapIndent
while remainingSoftWrapIndent > 0
indentToken = Math.min(remainingSoftWrapIndent, @tabLength)
rightSpecialTokens[rightTags.length] = SoftWrapIndent
rightTags.push(indentToken)
remainingSoftWrapIndent -= indentToken
rightTags.push(rightPrefix) if rightPrefix > 0
screenColumn += tag
# token is after split column
else
if specialToken = @specialTokens[index]
rightSpecialTokens[rightTags.length] = specialToken
rightTags.push(tag)
# tag represents the start or end of a scop
else if (tag % 2) is -1
if screenColumn < column
leftTags.push(tag)
rightOpenScopes.push(tag)
else
rightTags.push(tag)
else
if screenColumn < column
leftTags.push(tag)
rightOpenScopes.pop()
else
rightTags.push(tag)
splitBufferColumn = @bufferColumnForScreenColumn(column)
leftFragment = new TokenizedLine
leftFragment.tokenIterator = @tokenIterator
leftFragment.openScopes = @openScopes
leftFragment.text = leftText
leftFragment.tags = leftTags
leftFragment.specialTokens = leftSpecialTokens
leftFragment.startBufferColumn = @startBufferColumn
leftFragment.bufferDelta = splitBufferColumn - @startBufferColumn
leftFragment.ruleStack = @ruleStack
leftFragment.invisibles = @invisibles
leftFragment.lineEnding = null
leftFragment.indentLevel = @indentLevel
leftFragment.tabLength = @tabLength
leftFragment.firstNonWhitespaceIndex = Math.min(column, @firstNonWhitespaceIndex)
leftFragment.firstTrailingWhitespaceIndex = Math.min(column, @firstTrailingWhitespaceIndex)
rightFragment = new TokenizedLine
rightFragment.tokenIterator = @tokenIterator
rightFragment.openScopes = rightOpenScopes
rightFragment.text = rightText
rightFragment.tags = rightTags
rightFragment.specialTokens = rightSpecialTokens
rightFragment.startBufferColumn = splitBufferColumn
rightFragment.bufferDelta = @bufferDelta - splitBufferColumn
rightFragment.ruleStack = @ruleStack
rightFragment.invisibles = @invisibles
rightFragment.lineEnding = @lineEnding
rightFragment.indentLevel = @indentLevel
rightFragment.tabLength = @tabLength
rightFragment.endOfLineInvisibles = @endOfLineInvisibles
rightFragment.firstNonWhitespaceIndex = Math.max(softWrapIndent, @firstNonWhitespaceIndex - column + softWrapIndent)
rightFragment.firstTrailingWhitespaceIndex = Math.max(softWrapIndent, @firstTrailingWhitespaceIndex - column + softWrapIndent)
indentationTokens = @buildSoftWrapIndentationTokens(leftTokens[0], hangingIndent)
leftFragment = new TokenizedLine(
tokens: leftTokens
startBufferColumn: @startBufferColumn
ruleStack: @ruleStack
invisibles: @invisibles
lineEnding: null,
indentLevel: @indentLevel,
tabLength: @tabLength
)
rightFragment = new TokenizedLine(
tokens: indentationTokens.concat(rightTokens)
startBufferColumn: @bufferColumnForScreenColumn(column)
ruleStack: @ruleStack
invisibles: @invisibles
lineEnding: @lineEnding,
indentLevel: @indentLevel,
tabLength: @tabLength
)
[leftFragment, rightFragment]
isSoftWrapped: ->
@lineEnding is null
isColumnInsideSoftWrapIndentation: (targetColumn) ->
targetColumn < @getSoftWrapIndentationDelta()
isColumnInsideSoftWrapIndentation: (column) ->
return false if @softWrapIndentationTokens.length is 0
getSoftWrapIndentationDelta: ->
delta = 0
for tag, index in @tags
if tag >= 0
if @specialTokens[index] is SoftWrapIndent
delta += tag
else
break
delta
column < @softWrapIndentationDelta
getSoftWrapIndentationTokens: ->
_.select(@tokens, (token) -> token.isSoftWrapIndentation)
buildSoftWrapIndentationDelta: ->
_.reduce @softWrapIndentationTokens, ((acc, token) -> acc + token.screenDelta), 0
hasOnlySoftWrapIndentation: ->
@getSoftWrapIndentationDelta() is @text.length
@tokens.length is @softWrapIndentationTokens.length
tokenAtBufferColumn: (bufferColumn) ->
@tokens[@tokenIndexAtBufferColumn(bufferColumn)]
@@ -430,6 +210,58 @@ class TokenizedLine
delta = nextDelta
delta
breakOutAtomicTokens: (inputTokens) ->
outputTokens = []
breakOutLeadingSoftTabs = true
column = @startBufferColumn
for token in inputTokens
newTokens = token.breakOutAtomicTokens(@tabLength, breakOutLeadingSoftTabs, column)
column += newToken.value.length for newToken in newTokens
outputTokens.push(newTokens...)
breakOutLeadingSoftTabs = token.isOnlyWhitespace() if breakOutLeadingSoftTabs
outputTokens
markLeadingAndTrailingWhitespaceTokens: ->
@firstNonWhitespaceIndex = @text.search(NonWhitespaceRegex)
if @firstNonWhitespaceIndex > 0 and isPairedCharacter(@text, @firstNonWhitespaceIndex - 1)
@firstNonWhitespaceIndex--
firstTrailingWhitespaceIndex = @text.search(TrailingWhitespaceRegex)
@lineIsWhitespaceOnly = firstTrailingWhitespaceIndex is 0
index = 0
for token in @tokens
if index < @firstNonWhitespaceIndex
token.firstNonWhitespaceIndex = Math.min(index + token.value.length, @firstNonWhitespaceIndex - index)
# Only the *last* segment of a soft-wrapped line can have trailing whitespace
if @lineEnding? and (index + token.value.length > firstTrailingWhitespaceIndex)
token.firstTrailingWhitespaceIndex = Math.max(0, firstTrailingWhitespaceIndex - index)
index += token.value.length
return
substituteInvisibleCharacters: ->
invisibles = @invisibles
changedText = false
for token, i in @tokens
if token.isHardTab
if invisibles.tab
token.value = invisibles.tab + token.value.substring(invisibles.tab.length)
token.hasInvisibleCharacters = true
changedText = true
else
if invisibles.space
if token.hasLeadingWhitespace() and not token.isSoftWrapIndentation
token.value = token.value.replace LeadingWhitespaceRegex, (leadingWhitespace) ->
leadingWhitespace.replace RepeatedSpaceRegex, invisibles.space
token.hasInvisibleCharacters = true
changedText = true
if token.hasTrailingWhitespace()
token.value = token.value.replace TrailingWhitespaceRegex, (leadingWhitespace) ->
leadingWhitespace.replace RepeatedSpaceRegex, invisibles.space
token.hasInvisibleCharacters = true
changedText = true
@text = @buildText() if changedText
buildEndOfLineInvisibles: ->
@endOfLineInvisibles = []
{cr, eol} = @invisibles
@@ -442,13 +274,11 @@ class TokenizedLine
@endOfLineInvisibles.push(eol) if eol
isComment: ->
iterator = @getTokenIterator()
while iterator.next()
scopes = iterator.getScopes()
continue if scopes.length is 1
continue unless NonWhitespaceRegex.test(iterator.getText())
for scope in scopes
return true if CommentScopeRegex.test(scope)
for token in @tokens
continue if token.scopes.length is 1
continue if token.isOnlyWhitespace()
for scope in token.scopes
return true if _.contains(scope.split('.'), 'comment')
break
false
@@ -459,6 +289,42 @@ class TokenizedLine
@tokens[index]
getTokenCount: ->
count = 0
count++ for tag in @tags when tag >= 0
count
@tokens.length
bufferColumnForToken: (targetToken) ->
column = 0
for token in @tokens
return column if token is targetToken
column += token.bufferDelta
getScopeTree: ->
return @scopeTree if @scopeTree?
scopeStack = []
for token in @tokens
@updateScopeStack(scopeStack, token.scopes)
_.last(scopeStack).children.push(token)
@scopeTree = scopeStack[0]
@updateScopeStack(scopeStack, [])
@scopeTree
updateScopeStack: (scopeStack, desiredScopeDescriptor) ->
# Find a common prefix
for scope, i in desiredScopeDescriptor
break unless scopeStack[i]?.scope is desiredScopeDescriptor[i]
# Pop scopeDescriptor until we're at the common prefx
until scopeStack.length is i
poppedScope = scopeStack.pop()
_.last(scopeStack)?.children.push(poppedScope)
# Push onto common prefix until scopeStack equals desiredScopeDescriptor
for j in [i...desiredScopeDescriptor.length]
scopeStack.push(new Scope(desiredScopeDescriptor[j]))
return
class Scope
constructor: (@scope) ->
@children = []