diff --git a/package.json b/package.json index 7e6619ab8..15eb53b29 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "delegato": "^1", "emissary": "^1.3.3", "event-kit": "^1.2.0", - "first-mate": "^4.1.4", + "first-mate": "^3.1", "fs-plus": "^2.8.0", "fstream": "0.1.24", "fuzzaldrin": "^2.1", @@ -151,7 +151,7 @@ "language-ruby": "0.54.0", "language-ruby-on-rails": "0.21.0", "language-sass": "0.38.0", - "language-shellscript": "0.15.0", + "language-shellscript": "0.14.0", "language-source": "0.9.0", "language-sql": "0.15.0", "language-text": "0.6.0", diff --git a/spec/text-editor-presenter-spec.coffee b/spec/text-editor-presenter-spec.coffee index d15c4759d..7da866ab4 100644 --- a/spec/text-editor-presenter-spec.coffee +++ b/spec/text-editor-presenter-spec.coffee @@ -670,11 +670,7 @@ describe "TextEditorPresenter", -> expectValues lineStateForScreenRow(presenter, 4), { screenRow: 4 text: line4.text - tags: line4.tags - specialTokens: line4.specialTokens - firstNonWhitespaceIndex: line4.firstNonWhitespaceIndex - firstTrailingWhitespaceIndex: line4.firstTrailingWhitespaceIndex - invisibles: line4.invisibles + tokens: line4.tokens top: 10 * 4 } @@ -682,11 +678,7 @@ describe "TextEditorPresenter", -> expectValues lineStateForScreenRow(presenter, 5), { screenRow: 5 text: line5.text - tags: line5.tags - specialTokens: line5.specialTokens - firstNonWhitespaceIndex: line5.firstNonWhitespaceIndex - firstTrailingWhitespaceIndex: line5.firstTrailingWhitespaceIndex - invisibles: line5.invisibles + tokens: line5.tokens top: 10 * 5 } @@ -694,11 +686,7 @@ describe "TextEditorPresenter", -> expectValues lineStateForScreenRow(presenter, 6), { screenRow: 6 text: line6.text - tags: line6.tags - specialTokens: line6.specialTokens - firstNonWhitespaceIndex: line6.firstNonWhitespaceIndex - firstTrailingWhitespaceIndex: line6.firstTrailingWhitespaceIndex - invisibles: line6.invisibles + tokens: line6.tokens top: 10 * 6 } @@ -706,11 +694,7 @@ describe "TextEditorPresenter", -> expectValues lineStateForScreenRow(presenter, 7), { screenRow: 7 text: line7.text - tags: line7.tags - specialTokens: line7.specialTokens - firstNonWhitespaceIndex: line7.firstNonWhitespaceIndex - firstTrailingWhitespaceIndex: line7.firstTrailingWhitespaceIndex - invisibles: line7.invisibles + tokens: line7.tokens top: 10 * 7 } @@ -718,11 +702,7 @@ describe "TextEditorPresenter", -> expectValues lineStateForScreenRow(presenter, 8), { screenRow: 8 text: line8.text - tags: line8.tags - specialTokens: line8.specialTokens - firstNonWhitespaceIndex: line8.firstNonWhitespaceIndex - firstTrailingWhitespaceIndex: line8.firstTrailingWhitespaceIndex - invisibles: line8.invisibles + tokens: line8.tokens top: 10 * 8 } @@ -817,19 +797,19 @@ describe "TextEditorPresenter", -> line1 = editor.tokenizedLineForScreenRow(1) expectValues lineStateForScreenRow(presenter, 1), { text: line1.text - tags: line1.tags + tokens: line1.tokens } line2 = editor.tokenizedLineForScreenRow(2) expectValues lineStateForScreenRow(presenter, 2), { text: line2.text - tags: line2.tags + tokens: line2.tokens } line3 = editor.tokenizedLineForScreenRow(3) expectValues lineStateForScreenRow(presenter, 3), { text: line3.text - tags: line3.tags + tokens: line3.tokens } it "does not remove out-of-view lines corresponding to ::mouseWheelScreenRow until ::stoppedScrollingDelay elapses", -> diff --git a/spec/text-editor-spec.coffee b/spec/text-editor-spec.coffee index a845619ba..d1d311088 100644 --- a/spec/text-editor-spec.coffee +++ b/spec/text-editor-spec.coffee @@ -4110,9 +4110,8 @@ describe "TextEditor", -> runs -> grammar = atom.grammars.selectGrammar("text.js") - {line, tags} = grammar.tokenizeLine("var i; // http://github.com") + {tokens} = grammar.tokenizeLine("var i; // http://github.com") - tokens = atom.grammars.decodeTokens(line, tags) expect(tokens[0].value).toBe "var" expect(tokens[0].scopes).toEqual ["source.js", "storage.modifier.js"] diff --git a/spec/tokenized-buffer-spec.coffee b/spec/tokenized-buffer-spec.coffee index 45cc03a44..9d92335af 100644 --- a/spec/tokenized-buffer-spec.coffee +++ b/spec/tokenized-buffer-spec.coffee @@ -296,6 +296,14 @@ describe "TokenizedBuffer", -> expect(tokenizedBuffer.tokenizedLineForRow(5).ruleStack?).toBeTruthy() expect(tokenizedBuffer.tokenizedLineForRow(6).ruleStack?).toBeTruthy() + describe ".findOpeningBracket(closingBufferPosition)", -> + it "returns the position of the matching bracket, skipping any nested brackets", -> + expect(tokenizedBuffer.findOpeningBracket([9, 2])).toEqual [1, 29] + + describe ".findClosingBracket(startBufferPosition)", -> + it "returns the position of the matching bracket, skipping any nested brackets", -> + expect(tokenizedBuffer.findClosingBracket([1, 29])).toEqual [9, 2] + it "tokenizes leading whitespace based on the new tab length", -> expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[0].isAtomic).toBeTruthy() expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[0].value).toBe " " @@ -572,7 +580,7 @@ describe "TokenizedBuffer", -> describe "when the selector matches a run of multiple tokens at the position", -> it "returns the range covered by all contigous tokens (within a single line)", -> - expect(tokenizedBuffer.bufferRangeForScopeAtPosition('.meta.function', [1, 18])).toEqual [[1, 6], [1, 28]] + expect(tokenizedBuffer.bufferRangeForScopeAtPosition('.function', [1, 18])).toEqual [[1, 6], [1, 28]] describe "when the editor.tabLength config value changes", -> it "updates the tab length of the tokenized lines", -> @@ -689,6 +697,22 @@ describe "TokenizedBuffer", -> expect(line.tokens[0].firstNonWhitespaceIndex).toBe 2 expect(line.tokens[line.tokens.length - 1].firstTrailingWhitespaceIndex).toBe 0 + it "sets the ::firstNonWhitespaceIndex and ::firstTrailingWhitespaceIndex correctly when tokens are split for soft-wrapping", -> + atom.config.set("editor.showInvisibles", true) + atom.config.set("editor.invisibles", space: 'S') + buffer.setText(" token ") + fullyTokenize(tokenizedBuffer) + token = tokenizedBuffer.tokenizedLines[0].tokens[0] + + [leftToken, rightToken] = token.splitAt(1) + expect(leftToken.hasInvisibleCharacters).toBe true + expect(leftToken.firstNonWhitespaceIndex).toBe 1 + expect(leftToken.firstTrailingWhitespaceIndex).toBe null + + expect(leftToken.hasInvisibleCharacters).toBe true + expect(rightToken.firstNonWhitespaceIndex).toBe null + expect(rightToken.firstTrailingWhitespaceIndex).toBe 5 + describe ".indentLevel on tokenized lines", -> beforeEach -> buffer = atom.project.bufferForPathSync('sample.js') @@ -728,7 +752,7 @@ describe "TokenizedBuffer", -> it "updates empty line indent guides when the empty line is the last line", -> buffer.insert([12, 2], '\n') - # The newline and the tab need to be in two different operations to surface the bug + # The newline and he tab need to be in two different operations to surface the bug buffer.insert([12, 0], ' ') expect(tokenizedBuffer.tokenizedLineForRow(13).indentLevel).toBe 1 diff --git a/spec/tokenized-line-spec.coffee b/spec/tokenized-line-spec.coffee index 2914ec089..0da83c91c 100644 --- a/spec/tokenized-line-spec.coffee +++ b/spec/tokenized-line-spec.coffee @@ -17,3 +17,24 @@ describe "TokenizedLine", -> it "returns false when the line is not only whitespace", -> expect(editor.tokenizedLineForScreenRow(0).isOnlyWhitespace()).toBe false expect(editor.tokenizedLineForScreenRow(2).isOnlyWhitespace()).toBe false + + describe "::getScopeTree()", -> + it "returns a tree whose inner nodes are scopeDescriptor and whose leaf nodes are tokens in those scopeDescriptor", -> + [tokens, tokenIndex] = [] + + ensureValidScopeTree = (scopeTree, scopeDescriptor=[]) -> + if scopeTree.children? + for child in scopeTree.children + ensureValidScopeTree(child, scopeDescriptor.concat([scopeTree.scope])) + else + expect(scopeTree).toBe tokens[tokenIndex++] + expect(scopeDescriptor).toEqual scopeTree.scopes + + waitsForPromise -> + atom.project.open('coffee.coffee').then (o) -> editor = o + + runs -> + tokenIndex = 0 + tokens = editor.tokenizedLineForScreenRow(1).tokens + scopeTree = editor.tokenizedLineForScreenRow(1).getScopeTree() + ensureValidScopeTree(scopeTree) diff --git a/src/display-buffer.coffee b/src/display-buffer.coffee index b2460addc..26bf43dce 100644 --- a/src/display-buffer.coffee +++ b/src/display-buffer.coffee @@ -2,7 +2,6 @@ _ = require 'underscore-plus' Serializable = require 'serializable' {CompositeDisposable, Emitter} = require 'event-kit' {Point, Range} = require 'text-buffer' -Grim = require 'grim' TokenizedBuffer = require './tokenized-buffer' RowMap = require './row-map' Fold = require './fold' @@ -10,6 +9,7 @@ Model = require './model' Token = require './token' Decoration = require './decoration' Marker = require './marker' +Grim = require 'grim' class BufferToScreenConversionError extends Error constructor: (@message, @metadata) -> @@ -659,19 +659,16 @@ class DisplayBuffer extends Model top = targetRow * @lineHeightInPixels left = 0 column = 0 - - iterator = @tokenizedLineForScreenRow(targetRow).getTokenIterator() - while iterator.next() - charWidths = @getScopedCharWidths(iterator.getScopes()) + for token in @tokenizedLineForScreenRow(targetRow).tokens + charWidths = @getScopedCharWidths(token.scopes) valueIndex = 0 - value = iterator.getText() - while valueIndex < value.length - if iterator.isPairedCharacter() - char = value + while valueIndex < token.value.length + if token.hasPairedCharacter + char = token.value.substr(valueIndex, 2) charLength = 2 valueIndex += 2 else - char = value[valueIndex] + char = token.value[valueIndex] charLength = 1 valueIndex++ @@ -692,19 +689,16 @@ class DisplayBuffer extends Model left = 0 column = 0 - - iterator = @tokenizedLineForScreenRow(row).getTokenIterator() - while iterator.next() - charWidths = @getScopedCharWidths(iterator.getScopes()) - value = iterator.getText() + for token in @tokenizedLineForScreenRow(row).tokens + charWidths = @getScopedCharWidths(token.scopes) valueIndex = 0 - while valueIndex < value.length - if iterator.isPairedCharacter() - char = value + while valueIndex < token.value.length + if token.hasPairedCharacter + char = token.value.substr(valueIndex, 2) charLength = 2 valueIndex += 2 else - char = value[valueIndex] + char = token.value[valueIndex] charLength = 1 valueIndex++ diff --git a/src/language-mode.coffee b/src/language-mode.coffee index c9401550b..b5529a05e 100644 --- a/src/language-mode.coffee +++ b/src/language-mode.coffee @@ -242,9 +242,8 @@ class LanguageMode @suggestedIndentForTokenizedLineAtBufferRow(bufferRow, tokenizedLine, options) suggestedIndentForTokenizedLineAtBufferRow: (bufferRow, tokenizedLine, options) -> - iterator = tokenizedLine.getTokenIterator() - iterator.next() - scopeDescriptor = new ScopeDescriptor(scopes: iterator.getScopes()) + scopes = tokenizedLine.tokens[0].scopes + scopeDescriptor = new ScopeDescriptor({scopes}) currentIndentLevel = @editor.indentationForBufferRow(bufferRow) return currentIndentLevel unless increaseIndentRegex = @increaseIndentRegexForScopeDescriptor(scopeDescriptor) diff --git a/src/lines-component.coffee b/src/lines-component.coffee index 17c904e99..fbec40b79 100644 --- a/src/lines-component.coffee +++ b/src/lines-component.coffee @@ -4,13 +4,10 @@ _ = require 'underscore-plus' CursorsComponent = require './cursors-component' HighlightsComponent = require './highlights-component' -TokenIterator = require './token-iterator' DummyLineNode = $$(-> @div className: 'line', style: 'position: absolute; visibility: hidden;', => @span 'x')[0] AcceptFilter = {acceptNode: -> NodeFilter.FILTER_ACCEPT} WrapperDiv = document.createElement('div') -TokenTextEscapeRegex = /[&"'<>]/g -MaxTokenLength = 20000 cloneObject = (object) -> clone = {} @@ -22,7 +19,6 @@ class LinesComponent placeholderTextDiv: null constructor: ({@presenter, @hostElement, @useShadowDOM, visible}) -> - @tokenIterator = new TokenIterator @measuredLines = new Set @lineNodesByLineId = {} @screenRowsByLineId = {} @@ -171,116 +167,20 @@ class LinesComponent @buildEndOfLineHTML(id) or ' ' buildLineInnerHTML: (id) -> - lineState = @newState.lines[id] - {firstNonWhitespaceIndex, firstTrailingWhitespaceIndex, invisibles} = lineState - lineIsWhitespaceOnly = firstTrailingWhitespaceIndex is 0 - + {indentGuidesVisible} = @newState + {tokens, text, isOnlyWhitespace} = @newState.lines[id] innerHTML = "" - @tokenIterator.reset(lineState) - while @tokenIterator.next() - for scope in @tokenIterator.getScopeEnds() - innerHTML += "" - - for scope in @tokenIterator.getScopeStarts() - innerHTML += "" - - tokenStart = @tokenIterator.getScreenStart() - tokenEnd = @tokenIterator.getScreenEnd() - tokenText = @tokenIterator.getText() - isHardTab = @tokenIterator.isHardTab() - - if hasLeadingWhitespace = tokenStart < firstNonWhitespaceIndex - tokenFirstNonWhitespaceIndex = firstNonWhitespaceIndex - tokenStart - else - tokenFirstNonWhitespaceIndex = null - - if hasTrailingWhitespace = tokenEnd > firstTrailingWhitespaceIndex - tokenFirstTrailingWhitespaceIndex = Math.max(0, firstTrailingWhitespaceIndex - tokenStart) - else - tokenFirstTrailingWhitespaceIndex = null - - hasIndentGuide = - @newState.indentGuidesVisible and - (hasLeadingWhitespace or lineIsWhitespaceOnly) - - hasInvisibleCharacters = - (invisibles?.tab and isHardTab) or - (invisibles?.space and (hasLeadingWhitespace or hasTrailingWhitespace)) - - innerHTML += @buildTokenHTML(tokenText, isHardTab, tokenFirstNonWhitespaceIndex, tokenFirstTrailingWhitespaceIndex, hasIndentGuide, hasInvisibleCharacters) - - for scope in @tokenIterator.getScopeEnds() - innerHTML += "" - - for scope in @tokenIterator.getScopes() - innerHTML += "" + scopeStack = [] + for token in tokens + innerHTML += @updateScopeStack(scopeStack, token.scopes) + hasIndentGuide = indentGuidesVisible and (token.hasLeadingWhitespace() or (token.hasTrailingWhitespace() and isOnlyWhitespace)) + innerHTML += token.getValueAsHtml({hasIndentGuide}) + innerHTML += @popScope(scopeStack) while scopeStack.length > 0 innerHTML += @buildEndOfLineHTML(id) innerHTML - buildTokenHTML: (tokenText, isHardTab, firstNonWhitespaceIndex, firstTrailingWhitespaceIndex, hasIndentGuide, hasInvisibleCharacters) -> - if isHardTab - classes = 'hard-tab' - classes += ' leading-whitespace' if firstNonWhitespaceIndex? - classes += ' trailing-whitespace' if firstTrailingWhitespaceIndex? - classes += ' indent-guide' if hasIndentGuide - classes += ' invisible-character' if hasInvisibleCharacters - return "#{@escapeTokenText(tokenText)}" - else - startIndex = 0 - endIndex = tokenText.length - - leadingHtml = '' - trailingHtml = '' - - if firstNonWhitespaceIndex? - leadingWhitespace = tokenText.substring(0, firstNonWhitespaceIndex) - - classes = 'leading-whitespace' - classes += ' indent-guide' if hasIndentGuide - classes += ' invisible-character' if hasInvisibleCharacters - - leadingHtml = "#{leadingWhitespace}" - startIndex = firstNonWhitespaceIndex - - if firstTrailingWhitespaceIndex? - tokenIsOnlyWhitespace = firstTrailingWhitespaceIndex is 0 - trailingWhitespace = tokenText.substring(firstTrailingWhitespaceIndex) - - classes = 'trailing-whitespace' - classes += ' indent-guide' if hasIndentGuide and not firstNonWhitespaceIndex? and tokenIsOnlyWhitespace - classes += ' invisible-character' if hasInvisibleCharacters - - trailingHtml = "#{trailingWhitespace}" - - endIndex = firstTrailingWhitespaceIndex - - html = leadingHtml - if tokenText.length > MaxTokenLength - while startIndex < endIndex - html += "" + @escapeTokenText(tokenText, startIndex, startIndex + MaxTokenLength) + "" - startIndex += MaxTokenLength - else - html += @escapeTokenText(tokenText, startIndex, endIndex) - - html += trailingHtml - html - - escapeTokenText: (tokenText, startIndex, endIndex) -> - if startIndex? and endIndex? and startIndex > 0 or endIndex < tokenText.length - tokenText = tokenText.slice(startIndex, endIndex) - tokenText.replace(TokenTextEscapeRegex, @escapeTokenTextReplace) - - escapeTokenTextReplace: (match) -> - switch match - when '&' then '&' - when '"' then '"' - when "'" then ''' - when '<' then '<' - when '>' then '>' - else match - buildEndOfLineHTML: (id) -> {endOfLineInvisibles} = @newState.lines[id] @@ -290,6 +190,31 @@ class LinesComponent html += "#{invisible}" html + updateScopeStack: (scopeStack, desiredScopeDescriptor) -> + html = "" + + # Find a common prefix + for scope, i in desiredScopeDescriptor + break unless scopeStack[i] is desiredScopeDescriptor[i] + + # Pop scopeDescriptor until we're at the common prefx + until scopeStack.length is i + html += @popScope(scopeStack) + + # Push onto common prefix until scopeStack equals desiredScopeDescriptor + for j in [i...desiredScopeDescriptor.length] + html += @pushScope(scopeStack, desiredScopeDescriptor[j]) + + html + + popScope: (scopeStack) -> + scopeStack.pop() + "" + + pushScope: (scopeStack, scope) -> + scopeStack.push(scope) + "" + updateLineNode: (id) -> oldLineState = @oldState.lines[id] newLineState = @newState.lines[id] @@ -354,22 +279,19 @@ class LinesComponent iterator = null charIndex = 0 - @tokenIterator.reset(tokenizedLine) - while @tokenIterator.next() - scopes = @tokenIterator.getScopes() - text = @tokenIterator.getText() + for {value, scopes, hasPairedCharacter} in tokenizedLine.tokens charWidths = @presenter.getScopedCharacterWidths(scopes) - textIndex = 0 - while textIndex < text.length - if @tokenIterator.isPairedCharacter() - char = text + valueIndex = 0 + while valueIndex < value.length + if hasPairedCharacter + char = value.substr(valueIndex, 2) charLength = 2 - textIndex += 2 + valueIndex += 2 else - char = text[textIndex] + char = value[valueIndex] charLength = 1 - textIndex++ + valueIndex++ continue if char is '\0' diff --git a/src/special-token-symbols.coffee b/src/special-token-symbols.coffee deleted file mode 100644 index 06884b85f..000000000 --- a/src/special-token-symbols.coffee +++ /dev/null @@ -1,6 +0,0 @@ -module.exports = { - SoftTab: Symbol('SoftTab') - HardTab: Symbol('HardTab') - PairedCharacter: Symbol('PairedCharacter') - SoftWrapIndent: Symbol('SoftWrapIndent') -} diff --git a/src/text-editor-presenter.coffee b/src/text-editor-presenter.coffee index 3aea57f29..70c26a1a3 100644 --- a/src/text-editor-presenter.coffee +++ b/src/text-editor-presenter.coffee @@ -336,14 +336,9 @@ class TextEditorPresenter @state.content.lines[line.id] = screenRow: row text: line.text - openScopes: line.openScopes - tags: line.tags - specialTokens: line.specialTokens - firstNonWhitespaceIndex: line.firstNonWhitespaceIndex - firstTrailingWhitespaceIndex: line.firstTrailingWhitespaceIndex - invisibles: line.invisibles - endOfLineInvisibles: line.endOfLineInvisibles + tokens: line.tokens isOnlyWhitespace: line.isOnlyWhitespace() + endOfLineInvisibles: line.endOfLineInvisibles indentLevel: line.indentLevel tabLength: line.tabLength fold: line.fold @@ -1011,20 +1006,17 @@ class TextEditorPresenter top = targetRow * @lineHeight left = 0 column = 0 - - iterator = @model.tokenizedLineForScreenRow(targetRow).getTokenIterator() - while iterator.next() - characterWidths = @getScopedCharacterWidths(iterator.getScopes()) + for token in @model.tokenizedLineForScreenRow(targetRow).tokens + characterWidths = @getScopedCharacterWidths(token.scopes) valueIndex = 0 - text = iterator.getText() - while valueIndex < text.length - if iterator.isPairedCharacter() - char = text + while valueIndex < token.value.length + if token.hasPairedCharacter + char = token.value.substr(valueIndex, 2) charLength = 2 valueIndex += 2 else - char = text[valueIndex] + char = token.value[valueIndex] charLength = 1 valueIndex++ diff --git a/src/text-editor.coffee b/src/text-editor.coffee index 4489d82af..d2bd77522 100644 --- a/src/text-editor.coffee +++ b/src/text-editor.coffee @@ -2457,8 +2457,9 @@ class TextEditor extends Model # Extended: Determine if the given row is entirely a comment isBufferRowCommented: (bufferRow) -> if match = @lineTextForBufferRow(bufferRow).match(/\S/) + scopeDescriptor = @tokenForBufferPosition([bufferRow, match.index]).scopes @commentScopeSelector ?= new TextMateScopeSelector('comment.*') - @commentScopeSelector.matches(@scopeDescriptorForBufferPosition([bufferRow, match.index]).scopes) + @commentScopeSelector.matches(scopeDescriptor) logCursorScope: -> scopeDescriptor = @getLastCursor().getScopeDescriptor() diff --git a/src/token-iterator.coffee b/src/token-iterator.coffee deleted file mode 100644 index 202b044ba..000000000 --- a/src/token-iterator.coffee +++ /dev/null @@ -1,83 +0,0 @@ -{SoftTab, HardTab, PairedCharacter, SoftWrapIndent} = require './special-token-symbols' - -module.exports = -class TokenIterator - constructor: (line) -> - @reset(line) if line? - - reset: (@line) -> - @index = null - @bufferStart = @line.startBufferColumn - @bufferEnd = @bufferStart - @screenStart = 0 - @screenEnd = 0 - @scopes = @line.openScopes.map (id) -> atom.grammars.scopeForId(id) - @scopeStarts = @scopes.slice() - @scopeEnds = [] - this - - next: -> - {tags} = @line - - if @index? - @index++ - @scopeEnds.length = 0 - @scopeStarts.length = 0 - @bufferStart = @bufferEnd - @screenStart = @screenEnd - else - @index = 0 - - while @index < tags.length - tag = tags[@index] - if tag < 0 - if tag % 2 is 0 - @scopeEnds.push(atom.grammars.scopeForId(tag + 1)) - @scopes.pop() - else - scope = atom.grammars.scopeForId(tag) - @scopeStarts.push(scope) - @scopes.push(scope) - @index++ - else - if @isHardTab() - @screenEnd = @screenStart + tag - @bufferEnd = @bufferStart + 1 - else if @isSoftWrapIndentation() - @screenEnd = @screenStart + tag - @bufferEnd = @bufferStart + 0 - else - @screenEnd = @screenStart + tag - @bufferEnd = @bufferStart + tag - return true - - false - - getBufferStart: -> @bufferStart - getBufferEnd: -> @bufferEnd - - getScreenStart: -> @screenStart - getScreenEnd: -> @screenEnd - - getScopeStarts: -> @scopeStarts - getScopeEnds: -> @scopeEnds - - getScopes: -> @scopes - - getText: -> - @line.text.substring(@screenStart, @screenEnd) - - isSoftTab: -> - @line.specialTokens[@index] is SoftTab - - isHardTab: -> - @line.specialTokens[@index] is HardTab - - isSoftWrapIndentation: -> - @line.specialTokens[@index] is SoftWrapIndent - - isPairedCharacter: -> - @line.specialTokens[@index] is PairedCharacter - - isAtomic: -> - @isSoftTab() or @isHardTab() or @isSoftWrapIndentation() or @isPairedCharacter() diff --git a/src/token.coffee b/src/token.coffee index 60e8194f8..8aa4a8706 100644 --- a/src/token.coffee +++ b/src/token.coffee @@ -1,8 +1,13 @@ _ = require 'underscore-plus' +textUtils = require './text-utils' +WhitespaceRegexesByTabLength = {} +EscapeRegex = /[&"'<>]/g StartDotRegex = /^\.?/ WhitespaceRegex = /\S/ +MaxTokenLength = 20000 + # Represents a single unit of text as selected by a grammar. module.exports = class Token @@ -15,14 +20,10 @@ class Token firstTrailingWhitespaceIndex: null hasInvisibleCharacters: false - constructor: (properties) -> - {@value, @scopes, @isAtomic, @isHardTab, @bufferDelta} = properties - {@hasInvisibleCharacters, @hasPairedCharacter, @isSoftWrapIndentation} = properties - @firstNonWhitespaceIndex = properties.firstNonWhitespaceIndex ? null - @firstTrailingWhitespaceIndex = properties.firstTrailingWhitespaceIndex ? null - + constructor: ({@value, @scopes, @isAtomic, @bufferDelta, @isHardTab, @hasPairedCharacter, @isSoftWrapIndentation}) -> @screenDelta = @value.length @bufferDelta ?= @screenDelta + @hasPairedCharacter ?= textUtils.hasPairedCharacter(@value) isEqual: (other) -> # TODO: scopes is deprecated. This is here for the sake of lang package tests @@ -31,6 +32,126 @@ class Token isBracket: -> /^meta\.brace\b/.test(_.last(@scopes)) + splitAt: (splitIndex) -> + leftToken = new Token(value: @value.substring(0, splitIndex), scopes: @scopes) + rightToken = new Token(value: @value.substring(splitIndex), scopes: @scopes) + + if @firstNonWhitespaceIndex? + leftToken.firstNonWhitespaceIndex = Math.min(splitIndex, @firstNonWhitespaceIndex) + leftToken.hasInvisibleCharacters = @hasInvisibleCharacters + + if @firstNonWhitespaceIndex > splitIndex + rightToken.firstNonWhitespaceIndex = @firstNonWhitespaceIndex - splitIndex + rightToken.hasInvisibleCharacters = @hasInvisibleCharacters + + if @firstTrailingWhitespaceIndex? + rightToken.firstTrailingWhitespaceIndex = Math.max(0, @firstTrailingWhitespaceIndex - splitIndex) + rightToken.hasInvisibleCharacters = @hasInvisibleCharacters + + if @firstTrailingWhitespaceIndex < splitIndex + leftToken.firstTrailingWhitespaceIndex = @firstTrailingWhitespaceIndex + leftToken.hasInvisibleCharacters = @hasInvisibleCharacters + + [leftToken, rightToken] + + whitespaceRegexForTabLength: (tabLength) -> + WhitespaceRegexesByTabLength[tabLength] ?= new RegExp("([ ]{#{tabLength}})|(\t)|([^\t]+)", "g") + + breakOutAtomicTokens: (tabLength, breakOutLeadingSoftTabs, startColumn) -> + if @hasPairedCharacter + outputTokens = [] + column = startColumn + + for token in @breakOutPairedCharacters() + if token.isAtomic + outputTokens.push(token) + else + outputTokens.push(token.breakOutAtomicTokens(tabLength, breakOutLeadingSoftTabs, column)...) + breakOutLeadingSoftTabs = token.isOnlyWhitespace() if breakOutLeadingSoftTabs + column += token.value.length + + outputTokens + else + return [this] if @isAtomic + + if breakOutLeadingSoftTabs + return [this] unless /^[ ]|\t/.test(@value) + else + return [this] unless /\t/.test(@value) + + outputTokens = [] + regex = @whitespaceRegexForTabLength(tabLength) + column = startColumn + while match = regex.exec(@value) + [fullMatch, softTab, hardTab] = match + token = null + if softTab and breakOutLeadingSoftTabs + token = @buildSoftTabToken(tabLength) + else if hardTab + breakOutLeadingSoftTabs = false + token = @buildHardTabToken(tabLength, column) + else + breakOutLeadingSoftTabs = false + value = match[0] + token = new Token({value, @scopes}) + column += token.value.length + outputTokens.push(token) + + outputTokens + + breakOutPairedCharacters: -> + outputTokens = [] + index = 0 + nonPairStart = 0 + + while index < @value.length + if textUtils.isPairedCharacter(@value, index) + if nonPairStart isnt index + outputTokens.push(new Token({value: @value[nonPairStart...index], @scopes})) + outputTokens.push(@buildPairedCharacterToken(@value, index)) + index += 2 + nonPairStart = index + else + index++ + + if nonPairStart isnt index + outputTokens.push(new Token({value: @value[nonPairStart...index], @scopes})) + + outputTokens + + buildPairedCharacterToken: (value, index) -> + new Token( + value: value[index..index + 1] + scopes: @scopes + isAtomic: true + hasPairedCharacter: true + ) + + buildHardTabToken: (tabLength, column) -> + @buildTabToken(tabLength, true, column) + + buildSoftTabToken: (tabLength) -> + @buildTabToken(tabLength, false, 0) + + buildTabToken: (tabLength, isHardTab, column=0) -> + tabStop = tabLength - (column % tabLength) + new Token( + value: _.multiplyString(" ", tabStop) + scopes: @scopes + bufferDelta: if isHardTab then 1 else tabStop + isAtomic: true + isHardTab: isHardTab + ) + + buildSoftWrapIndentationToken: (length) -> + new Token( + value: _.multiplyString(" ", length), + scopes: @scopes, + bufferDelta: 0, + isAtomic: true, + isSoftWrapIndentation: true + ) + isOnlyWhitespace: -> not WhitespaceRegex.test(@value) @@ -40,6 +161,72 @@ class Token scopeClasses = scope.split('.') _.isSubset(targetClasses, scopeClasses) + getValueAsHtml: ({hasIndentGuide}) -> + if @isHardTab + classes = 'hard-tab' + classes += ' leading-whitespace' if @hasLeadingWhitespace() + classes += ' trailing-whitespace' if @hasTrailingWhitespace() + classes += ' indent-guide' if hasIndentGuide + classes += ' invisible-character' if @hasInvisibleCharacters + html = "#{@escapeString(@value)}" + else + startIndex = 0 + endIndex = @value.length + + leadingHtml = '' + trailingHtml = '' + + if @hasLeadingWhitespace() + leadingWhitespace = @value.substring(0, @firstNonWhitespaceIndex) + + classes = 'leading-whitespace' + classes += ' indent-guide' if hasIndentGuide + classes += ' invisible-character' if @hasInvisibleCharacters + + leadingHtml = "#{leadingWhitespace}" + startIndex = @firstNonWhitespaceIndex + + if @hasTrailingWhitespace() + tokenIsOnlyWhitespace = @firstTrailingWhitespaceIndex is 0 + trailingWhitespace = @value.substring(@firstTrailingWhitespaceIndex) + + classes = 'trailing-whitespace' + classes += ' indent-guide' if hasIndentGuide and not @hasLeadingWhitespace() and tokenIsOnlyWhitespace + classes += ' invisible-character' if @hasInvisibleCharacters + + trailingHtml = "#{trailingWhitespace}" + + endIndex = @firstTrailingWhitespaceIndex + + html = leadingHtml + if @value.length > MaxTokenLength + while startIndex < endIndex + html += "" + @escapeString(@value, startIndex, startIndex + MaxTokenLength) + "" + startIndex += MaxTokenLength + else + html += @escapeString(@value, startIndex, endIndex) + + html += trailingHtml + html + + escapeString: (str, startIndex, endIndex) -> + strLength = str.length + + startIndex ?= 0 + endIndex ?= strLength + + str = str.slice(startIndex, endIndex) if startIndex > 0 or endIndex < strLength + str.replace(EscapeRegex, @escapeStringReplace) + + escapeStringReplace: (match) -> + switch match + when '&' then '&' + when '"' then '"' + when "'" then ''' + when '<' then '<' + when '>' then '>' + else match + hasLeadingWhitespace: -> @firstNonWhitespaceIndex? and @firstNonWhitespaceIndex > 0 diff --git a/src/tokenized-buffer.coffee b/src/tokenized-buffer.coffee index 60ebe16f0..6d8f0c018 100644 --- a/src/tokenized-buffer.coffee +++ b/src/tokenized-buffer.coffee @@ -1,11 +1,9 @@ _ = require 'underscore-plus' {CompositeDisposable, Emitter} = require 'event-kit' {Point, Range} = require 'text-buffer' -{ScopeSelector} = require 'first-mate' Serializable = require 'serializable' Model = require './model' TokenizedLine = require './tokenized-line' -TokenIterator = require './token-iterator' Token = require './token' ScopeDescriptor = require './scope-descriptor' Grim = require 'grim' @@ -27,7 +25,6 @@ class TokenizedBuffer extends Model constructor: ({@buffer, @tabLength, @ignoreInvisibles}) -> @emitter = new Emitter @disposables = new CompositeDisposable - @tokenIterator = new TokenIterator @disposables.add atom.grammars.onDidAddGrammar(@grammarAddedOrUpdated) @disposables.add atom.grammars.onDidUpdateGrammar(@grammarAddedOrUpdated) @@ -170,7 +167,7 @@ class TokenizedBuffer extends Model row = startRow loop previousStack = @stackForRow(row) - @tokenizedLines[row] = @buildTokenizedLineForRow(row, @stackForRow(row - 1), @openScopesForRow(row)) + @tokenizedLines[row] = @buildTokenizedLineForRow(row, @stackForRow(row - 1)) if --rowsRemaining is 0 filledRegion = false endRow = row @@ -230,7 +227,7 @@ class TokenizedBuffer extends Model @updateInvalidRows(start, end, delta) previousEndStack = @stackForRow(end) # used in spill detection below - newTokenizedLines = @buildTokenizedLinesForRows(start, end + delta, @stackForRow(start - 1), @openScopesForRow(start)) + newTokenizedLines = @buildTokenizedLinesForRows(start, end + delta, @stackForRow(start - 1)) _.spliceWithArray(@tokenizedLines, start, end - start + 1, newTokenizedLines) start = @retokenizeWhitespaceRowsIfIndentLevelChanged(start - 1, -1) @@ -251,7 +248,7 @@ class TokenizedBuffer extends Model line = @tokenizedLines[row] if line?.isOnlyWhitespace() and @indentLevelForRow(row) isnt line.indentLevel while line?.isOnlyWhitespace() - @tokenizedLines[row] = @buildTokenizedLineForRow(row, @stackForRow(row - 1), @openScopesForRow(row)) + @tokenizedLines[row] = @buildTokenizedLineForRow(row, @stackForRow(row - 1)) row += increment line = @tokenizedLines[row] @@ -293,18 +290,16 @@ class TokenizedBuffer extends Model @tokenizedLineForRow(row).isComment() and @tokenizedLineForRow(nextRow).isComment() - buildTokenizedLinesForRows: (startRow, endRow, startingStack, startingopenScopes) -> + buildTokenizedLinesForRows: (startRow, endRow, startingStack) -> ruleStack = startingStack - openScopes = startingopenScopes stopTokenizingAt = startRow + @chunkSize tokenizedLines = for row in [startRow..endRow] if (ruleStack or row is 0) and row < stopTokenizingAt - tokenizedLine = @buildTokenizedLineForRow(row, ruleStack, openScopes) - ruleStack = tokenizedLine.ruleStack - openScopes = @scopesFromTags(openScopes, tokenizedLine.tags) + screenLine = @buildTokenizedLineForRow(row, ruleStack) + ruleStack = screenLine.ruleStack else - tokenizedLine = @buildPlaceholderTokenizedLineForRow(row, openScopes) - tokenizedLine + screenLine = @buildPlaceholderTokenizedLineForRow(row) + screenLine if endRow >= stopTokenizingAt @invalidateRow(stopTokenizingAt) @@ -316,23 +311,22 @@ class TokenizedBuffer extends Model @buildPlaceholderTokenizedLineForRow(row) for row in [startRow..endRow] buildPlaceholderTokenizedLineForRow: (row) -> - openScopes = [@grammar.startIdForScope(@grammar.scopeName)] - text = @buffer.lineForRow(row) - tags = [text.length] + line = @buffer.lineForRow(row) + tokens = [new Token(value: line, scopes: [@grammar.scopeName])] tabLength = @getTabLength() indentLevel = @indentLevelForRow(row) lineEnding = @buffer.lineEndingForRow(row) - new TokenizedLine({openScopes, text, tags, tabLength, indentLevel, invisibles: @getInvisiblesToShow(), lineEnding, @tokenIterator}) + new TokenizedLine({tokens, tabLength, indentLevel, invisibles: @getInvisiblesToShow(), lineEnding}) - buildTokenizedLineForRow: (row, ruleStack, openScopes) -> - @buildTokenizedLineForRowWithText(row, @buffer.lineForRow(row), ruleStack, openScopes) + buildTokenizedLineForRow: (row, ruleStack) -> + @buildTokenizedLineForRowWithText(row, @buffer.lineForRow(row), ruleStack) - buildTokenizedLineForRowWithText: (row, text, ruleStack = @stackForRow(row - 1), openScopes = @openScopesForRow(row)) -> + buildTokenizedLineForRowWithText: (row, line, ruleStack = @stackForRow(row - 1)) -> lineEnding = @buffer.lineEndingForRow(row) tabLength = @getTabLength() indentLevel = @indentLevelForRow(row) - {tags, ruleStack} = @grammar.tokenizeLine(text, ruleStack, row is 0, false) - new TokenizedLine({openScopes, text, tags, ruleStack, tabLength, lineEnding, indentLevel, invisibles: @getInvisiblesToShow(), @tokenIterator}) + {tokens, ruleStack} = @grammar.tokenizeLine(line, ruleStack, row is 0) + new TokenizedLine({tokens, ruleStack, tabLength, lineEnding, indentLevel, invisibles: @getInvisiblesToShow()}) getInvisiblesToShow: -> if @configSettings.showInvisibles and not @ignoreInvisibles @@ -346,25 +340,6 @@ class TokenizedBuffer extends Model stackForRow: (bufferRow) -> @tokenizedLines[bufferRow]?.ruleStack - openScopesForRow: (bufferRow) -> - if bufferRow > 0 - precedingLine = @tokenizedLines[bufferRow - 1] - @scopesFromTags(precedingLine.openScopes, precedingLine.tags) - else - [] - - scopesFromTags: (startingScopes, tags) -> - scopes = startingScopes.slice() - for tag in tags when tag < 0 - if (tag % 2) is -1 - scopes.push(tag) - else - expectedScope = tag + 1 - poppedScope = scopes.pop() - unless poppedScope is expectedScope - throw new Error("Encountered an invalid scope end id. Popped #{poppedScope}, expected to pop #{expectedScope}.") - scopes - indentLevelForRow: (bufferRow) -> line = @buffer.lineForRow(bufferRow) indentLevel = 0 @@ -401,20 +376,7 @@ class TokenizedBuffer extends Model 0 scopeDescriptorForPosition: (position) -> - {row, column} = Point.fromObject(position) - - iterator = @tokenizedLines[row].getTokenIterator() - while iterator.next() - if iterator.getScreenEnd() > column - scopes = iterator.getScopes() - break - - # rebuild scope of last token if we iterated off the end - unless scopes? - scopes = iterator.getScopes() - scopes.push(iterator.getScopeEnds().reverse()...) - - new ScopeDescriptor({scopes}) + new ScopeDescriptor(scopes: @tokenForPosition(position).scopes) tokenForPosition: (position) -> {row, column} = Point.fromObject(position) @@ -426,53 +388,85 @@ class TokenizedBuffer extends Model new Point(row, column) bufferRangeForScopeAtPosition: (selector, position) -> - selector = new ScopeSelector(selector.replace(/^\./, '')) position = Point.fromObject(position) + tokenizedLine = @tokenizedLines[position.row] + startIndex = tokenizedLine.tokenIndexAtBufferColumn(position.column) - {openScopes, tags} = @tokenizedLines[position.row] - scopes = openScopes.map (tag) -> atom.grammars.scopeForId(tag) + for index in [startIndex..0] + token = tokenizedLine.tokenAtIndex(index) + break unless token.matchesScopeSelector(selector) + firstToken = token - startColumn = 0 - for tag, tokenIndex in tags - if tag < 0 - if tag % 2 is -1 - scopes.push(atom.grammars.scopeForId(tag)) - else - scopes.pop() - else - endColumn = startColumn + tag - if endColumn > position.column - break - else - startColumn = endColumn + for index in [startIndex...tokenizedLine.getTokenCount()] + token = tokenizedLine.tokenAtIndex(index) + break unless token.matchesScopeSelector(selector) + lastToken = token - return unless selector.matches(scopes) + return unless firstToken? and lastToken? - startScopes = scopes.slice() - for startTokenIndex in [(tokenIndex - 1)..0] by -1 - tag = tags[startTokenIndex] - if tag < 0 - if tag % 2 is -1 - startScopes.pop() - else - startScopes.push(atom.grammars.scopeForId(tag)) - else - break unless selector.matches(startScopes) - startColumn -= tag + startColumn = tokenizedLine.bufferColumnForToken(firstToken) + endColumn = tokenizedLine.bufferColumnForToken(lastToken) + lastToken.bufferDelta + new Range([position.row, startColumn], [position.row, endColumn]) - endScopes = scopes.slice() - for endTokenIndex in [(tokenIndex + 1)...tags.length] by 1 - tag = tags[endTokenIndex] - if tag < 0 - if tag % 2 is -1 - endScopes.push(atom.grammars.scopeForId(tag)) - else - endScopes.pop() - else - break unless selector.matches(endScopes) - endColumn += tag + iterateTokensInBufferRange: (bufferRange, iterator) -> + bufferRange = Range.fromObject(bufferRange) + {start, end} = bufferRange - new Range(new Point(position.row, startColumn), new Point(position.row, endColumn)) + keepLooping = true + stop = -> keepLooping = false + + for bufferRow in [start.row..end.row] + bufferColumn = 0 + for token in @tokenizedLines[bufferRow].tokens + startOfToken = new Point(bufferRow, bufferColumn) + iterator(token, startOfToken, {stop}) if bufferRange.containsPoint(startOfToken) + return unless keepLooping + bufferColumn += token.bufferDelta + + backwardsIterateTokensInBufferRange: (bufferRange, iterator) -> + bufferRange = Range.fromObject(bufferRange) + {start, end} = bufferRange + + keepLooping = true + stop = -> keepLooping = false + + for bufferRow in [end.row..start.row] + bufferColumn = @buffer.lineLengthForRow(bufferRow) + for token in new Array(@tokenizedLines[bufferRow].tokens...).reverse() + bufferColumn -= token.bufferDelta + startOfToken = new Point(bufferRow, bufferColumn) + iterator(token, startOfToken, {stop}) if bufferRange.containsPoint(startOfToken) + return unless keepLooping + + findOpeningBracket: (startBufferPosition) -> + range = [[0,0], startBufferPosition] + position = null + depth = 0 + @backwardsIterateTokensInBufferRange range, (token, startPosition, {stop}) -> + if token.isBracket() + if token.value is '}' + depth++ + else if token.value is '{' + depth-- + if depth is 0 + position = startPosition + stop() + position + + findClosingBracket: (startBufferPosition) -> + range = [startBufferPosition, @buffer.getEndPosition()] + position = null + depth = 0 + @iterateTokensInBufferRange range, (token, startPosition, {stop}) -> + if token.isBracket() + if token.value is '{' + depth++ + else if token.value is '}' + depth-- + if depth is 0 + position = startPosition + stop() + position # Gets the row number of the last line. # diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index 45af81e57..b81d972a0 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -1,13 +1,10 @@ _ = require 'underscore-plus' {isPairedCharacter} = require './text-utils' -Token = require './token' -{SoftTab, HardTab, PairedCharacter, SoftWrapIndent} = require './special-token-symbols' NonWhitespaceRegex = /\S/ LeadingWhitespaceRegex = /^\s*/ TrailingWhitespaceRegex = /\s*$/ RepeatedSpaceRegex = /[ ]/g -CommentScopeRegex = /(\b|\.)comment/ idCounter = 1 module.exports = @@ -17,181 +14,32 @@ class TokenizedLine firstNonWhitespaceIndex: 0 foldable: false - constructor: (properties) -> - @id = idCounter++ - - return unless properties? - - @specialTokens = {} - {@openScopes, @text, @tags, @lineEnding, @ruleStack, @tokenIterator} = properties - {@startBufferColumn, @fold, @tabLength, @indentLevel, @invisibles} = properties - + constructor: ({tokens, @lineEnding, @ruleStack, @startBufferColumn, @fold, @tabLength, @indentLevel, @invisibles}) -> @startBufferColumn ?= 0 - @bufferDelta = @text.length + @tokens = @breakOutAtomicTokens(tokens) + @text = @buildText() + @bufferDelta = @buildBufferDelta() + @softWrapIndentationTokens = @getSoftWrapIndentationTokens() + @softWrapIndentationDelta = @buildSoftWrapIndentationDelta() - @transformContent() - @buildEndOfLineInvisibles() if @invisibles? and @lineEnding? + @id = idCounter++ + @markLeadingAndTrailingWhitespaceTokens() + if @invisibles + @substituteInvisibleCharacters() + @buildEndOfLineInvisibles() if @lineEnding? - transformContent: -> - text = '' - bufferColumn = 0 - screenColumn = 0 - tokenIndex = 0 - tokenOffset = 0 - firstNonWhitespaceColumn = null - lastNonWhitespaceColumn = null + buildText: -> + text = "" + text += token.value for token in @tokens + text - while bufferColumn < @text.length - # advance to next token if we've iterated over its length - if tokenOffset is @tags[tokenIndex] - tokenIndex++ - tokenOffset = 0 - - # advance to next token tag - tokenIndex++ while @tags[tokenIndex] < 0 - - character = @text[bufferColumn] - - # split out unicode surrogate pairs - if isPairedCharacter(@text, bufferColumn) - prefix = tokenOffset - suffix = @tags[tokenIndex] - tokenOffset - 2 - splitTokens = [] - splitTokens.push(prefix) if prefix > 0 - splitTokens.push(2) - splitTokens.push(suffix) if suffix > 0 - - @tags.splice(tokenIndex, 1, splitTokens...) - - firstNonWhitespaceColumn ?= screenColumn - lastNonWhitespaceColumn = screenColumn + 1 - - text += @text.substr(bufferColumn, 2) - screenColumn += 2 - bufferColumn += 2 - - tokenIndex++ if prefix > 0 - @specialTokens[tokenIndex] = PairedCharacter - tokenIndex++ - tokenOffset = 0 - - # split out leading soft tabs - else if character is ' ' - if firstNonWhitespaceColumn? - text += ' ' - else - if (screenColumn + 1) % @tabLength is 0 - @specialTokens[tokenIndex] = SoftTab - suffix = @tags[tokenIndex] - @tabLength - @tags.splice(tokenIndex, 1, @tabLength) - @tags.splice(tokenIndex + 1, 0, suffix) if suffix > 0 - text += @invisibles?.space ? ' ' - - screenColumn++ - bufferColumn++ - tokenOffset++ - - # expand hard tabs to the next tab stop - else if character is '\t' - tabLength = @tabLength - (screenColumn % @tabLength) - if @invisibles?.tab - text += @invisibles.tab - else - text += ' ' - text += ' ' for i in [1...tabLength] by 1 - - prefix = tokenOffset - suffix = @tags[tokenIndex] - tokenOffset - 1 - splitTokens = [] - splitTokens.push(prefix) if prefix > 0 - splitTokens.push(tabLength) - splitTokens.push(suffix) if suffix > 0 - - @tags.splice(tokenIndex, 1, splitTokens...) - - screenColumn += tabLength - bufferColumn++ - - tokenIndex++ if prefix > 0 - @specialTokens[tokenIndex] = HardTab - tokenIndex++ - tokenOffset = 0 - - # continue past any other character - else - firstNonWhitespaceColumn ?= screenColumn - lastNonWhitespaceColumn = screenColumn - - text += character - screenColumn++ - bufferColumn++ - tokenOffset++ - - @text = text - - @firstNonWhitespaceIndex = firstNonWhitespaceColumn - if lastNonWhitespaceColumn? - if lastNonWhitespaceColumn + 1 < @text.length - @firstTrailingWhitespaceIndex = lastNonWhitespaceColumn + 1 - if @invisibles?.space - @text = - @text.substring(0, @firstTrailingWhitespaceIndex) + - @text.substring(@firstTrailingWhitespaceIndex) - .replace(RepeatedSpaceRegex, @invisibles.space) - else - @lineIsWhitespaceOnly = true - @firstTrailingWhitespaceIndex = 0 - - getTokenIterator: -> @tokenIterator.reset(this) - - Object.defineProperty @prototype, 'tokens', get: -> - iterator = @getTokenIterator() - tokens = [] - - while iterator.next() - properties = { - value: iterator.getText() - scopes: iterator.getScopes().slice() - isAtomic: iterator.isAtomic() - isHardTab: iterator.isHardTab() - hasPairedCharacter: iterator.isPairedCharacter() - isSoftWrapIndentation: iterator.isSoftWrapIndentation() - } - - if iterator.isHardTab() - properties.bufferDelta = 1 - properties.hasInvisibleCharacters = true if @invisibles?.tab - - if iterator.getScreenStart() < @firstNonWhitespaceIndex - properties.firstNonWhitespaceIndex = - Math.min(@firstNonWhitespaceIndex, iterator.getScreenEnd()) - iterator.getScreenStart() - properties.hasInvisibleCharacters = true if @invisibles?.space - - if @lineEnding? and iterator.getScreenEnd() > @firstTrailingWhitespaceIndex - properties.firstTrailingWhitespaceIndex = - Math.max(0, @firstTrailingWhitespaceIndex - iterator.getScreenStart()) - properties.hasInvisibleCharacters = true if @invisibles?.space - - tokens.push(new Token(properties)) - - tokens + buildBufferDelta: -> + delta = 0 + delta += token.bufferDelta for token in @tokens + delta copy: -> - copy = new TokenizedLine - copy.tokenIterator = @tokenIterator - copy.indentLevel = @indentLevel - copy.openScopes = @openScopes - copy.text = @text - copy.tags = @tags - copy.specialTokens = @specialTokens - copy.firstNonWhitespaceIndex = @firstNonWhitespaceIndex - copy.firstTrailingWhitespaceIndex = @firstTrailingWhitespaceIndex - copy.lineEnding = @lineEnding - copy.endOfLineInvisibles = @endOfLineInvisibles - copy.ruleStack = @ruleStack - copy.startBufferColumn = @startBufferColumn - copy.fold = @fold - copy + new TokenizedLine({@tokens, @lineEnding, @ruleStack, @startBufferColumn, @fold}) # This clips a given screen column to a valid column that's within the line # and not in the middle of any atomic tokens. @@ -204,58 +52,49 @@ class TokenizedLine # # Returns a {Number} representing the clipped column. clipScreenColumn: (column, options={}) -> - return 0 if @tags.length is 0 + return 0 if @tokens.length is 0 {clip} = options column = Math.min(column, @getMaxScreenColumn()) tokenStartColumn = 0 + for token in @tokens + break if tokenStartColumn + token.screenDelta > column + tokenStartColumn += token.screenDelta - iterator = @getTokenIterator() - while iterator.next() - break if iterator.getScreenEnd() > column - - if iterator.isSoftWrapIndentation() - iterator.next() while iterator.isSoftWrapIndentation() - iterator.getScreenStart() - else if iterator.isAtomic() and iterator.getScreenStart() < column + if @isColumnInsideSoftWrapIndentation(tokenStartColumn) + @softWrapIndentationDelta + else if token.isAtomic and tokenStartColumn < column if clip is 'forward' - iterator.getScreenEnd() + tokenStartColumn + token.screenDelta else if clip is 'backward' - iterator.getScreenStart() + tokenStartColumn else #'closest' - if column > ((iterator.getScreenStart() + iterator.getScreenEnd()) / 2) - iterator.getScreenEnd() + if column > tokenStartColumn + (token.screenDelta / 2) + tokenStartColumn + token.screenDelta else - iterator.getScreenStart() + tokenStartColumn else column - screenColumnForBufferColumn: (targetBufferColumn, options) -> - iterator = @getTokenIterator() - while iterator.next() - tokenBufferStart = iterator.getBufferStart() - tokenBufferEnd = iterator.getBufferEnd() - if tokenBufferStart <= targetBufferColumn < tokenBufferEnd - overshoot = targetBufferColumn - tokenBufferStart - return Math.min( - iterator.getScreenStart() + overshoot, - iterator.getScreenEnd() - ) - iterator.getScreenEnd() + screenColumnForBufferColumn: (bufferColumn, options) -> + bufferColumn = bufferColumn - @startBufferColumn + screenColumn = 0 + currentBufferColumn = 0 + for token in @tokens + break if currentBufferColumn + token.bufferDelta > bufferColumn + screenColumn += token.screenDelta + currentBufferColumn += token.bufferDelta + @clipScreenColumn(screenColumn + (bufferColumn - currentBufferColumn)) - bufferColumnForScreenColumn: (targetScreenColumn) -> - iterator = @getTokenIterator() - while iterator.next() - tokenScreenStart = iterator.getScreenStart() - tokenScreenEnd = iterator.getScreenEnd() - if tokenScreenStart <= targetScreenColumn < tokenScreenEnd - overshoot = targetScreenColumn - tokenScreenStart - return Math.min( - iterator.getBufferStart() + overshoot, - iterator.getBufferEnd() - ) - iterator.getBufferEnd() + bufferColumnForScreenColumn: (screenColumn, options) -> + bufferColumn = @startBufferColumn + currentScreenColumn = 0 + for token in @tokens + break if currentScreenColumn + token.screenDelta > screenColumn + bufferColumn += token.bufferDelta + currentScreenColumn += token.screenDelta + bufferColumn + (screenColumn - currentScreenColumn) getMaxScreenColumn: -> if @fold @@ -289,128 +128,69 @@ class TokenizedLine return maxColumn + buildSoftWrapIndentationTokens: (token, hangingIndent) -> + totalIndentSpaces = (@indentLevel * @tabLength) + hangingIndent + indentTokens = [] + while totalIndentSpaces > 0 + tokenLength = Math.min(@tabLength, totalIndentSpaces) + indentToken = token.buildSoftWrapIndentationToken(tokenLength) + indentTokens.push(indentToken) + totalIndentSpaces -= tokenLength + + indentTokens + softWrapAt: (column, hangingIndent) -> - return [null, this] if column is 0 + return [new TokenizedLine([], '', [0, 0], [0, 0]), this] if column is 0 - leftText = @text.substring(0, column) - rightText = @text.substring(column) + rightTokens = new Array(@tokens...) + leftTokens = [] + leftScreenColumn = 0 - leftTags = [] - rightTags = [] + while leftScreenColumn < column + if leftScreenColumn + rightTokens[0].screenDelta > column + rightTokens[0..0] = rightTokens[0].splitAt(column - leftScreenColumn) + nextToken = rightTokens.shift() + leftScreenColumn += nextToken.screenDelta + leftTokens.push nextToken - leftSpecialTokens = {} - rightSpecialTokens = {} - - rightOpenScopes = @openScopes.slice() - - screenColumn = 0 - - for tag, index in @tags - # tag represents a token - if tag >= 0 - # token ends before the soft wrap column - if screenColumn + tag <= column - if specialToken = @specialTokens[index] - leftSpecialTokens[index] = specialToken - leftTags.push(tag) - screenColumn += tag - - # token starts before and ends after the split column - else if screenColumn <= column - leftSuffix = column - screenColumn - rightPrefix = screenColumn + tag - column - - leftTags.push(leftSuffix) if leftSuffix > 0 - - softWrapIndent = @indentLevel * @tabLength + (hangingIndent ? 0) - for i in [0...softWrapIndent] by 1 - rightText = ' ' + rightText - remainingSoftWrapIndent = softWrapIndent - while remainingSoftWrapIndent > 0 - indentToken = Math.min(remainingSoftWrapIndent, @tabLength) - rightSpecialTokens[rightTags.length] = SoftWrapIndent - rightTags.push(indentToken) - remainingSoftWrapIndent -= indentToken - - rightTags.push(rightPrefix) if rightPrefix > 0 - - screenColumn += tag - - # token is after split column - else - if specialToken = @specialTokens[index] - rightSpecialTokens[rightTags.length] = specialToken - rightTags.push(tag) - - # tag represents the start or end of a scop - else if (tag % 2) is -1 - if screenColumn < column - leftTags.push(tag) - rightOpenScopes.push(tag) - else - rightTags.push(tag) - else - if screenColumn < column - leftTags.push(tag) - rightOpenScopes.pop() - else - rightTags.push(tag) - - splitBufferColumn = @bufferColumnForScreenColumn(column) - - leftFragment = new TokenizedLine - leftFragment.tokenIterator = @tokenIterator - leftFragment.openScopes = @openScopes - leftFragment.text = leftText - leftFragment.tags = leftTags - leftFragment.specialTokens = leftSpecialTokens - leftFragment.startBufferColumn = @startBufferColumn - leftFragment.bufferDelta = splitBufferColumn - @startBufferColumn - leftFragment.ruleStack = @ruleStack - leftFragment.invisibles = @invisibles - leftFragment.lineEnding = null - leftFragment.indentLevel = @indentLevel - leftFragment.tabLength = @tabLength - leftFragment.firstNonWhitespaceIndex = Math.min(column, @firstNonWhitespaceIndex) - leftFragment.firstTrailingWhitespaceIndex = Math.min(column, @firstTrailingWhitespaceIndex) - - rightFragment = new TokenizedLine - rightFragment.tokenIterator = @tokenIterator - rightFragment.openScopes = rightOpenScopes - rightFragment.text = rightText - rightFragment.tags = rightTags - rightFragment.specialTokens = rightSpecialTokens - rightFragment.startBufferColumn = splitBufferColumn - rightFragment.bufferDelta = @bufferDelta - splitBufferColumn - rightFragment.ruleStack = @ruleStack - rightFragment.invisibles = @invisibles - rightFragment.lineEnding = @lineEnding - rightFragment.indentLevel = @indentLevel - rightFragment.tabLength = @tabLength - rightFragment.endOfLineInvisibles = @endOfLineInvisibles - rightFragment.firstNonWhitespaceIndex = Math.max(softWrapIndent, @firstNonWhitespaceIndex - column + softWrapIndent) - rightFragment.firstTrailingWhitespaceIndex = Math.max(softWrapIndent, @firstTrailingWhitespaceIndex - column + softWrapIndent) + indentationTokens = @buildSoftWrapIndentationTokens(leftTokens[0], hangingIndent) + leftFragment = new TokenizedLine( + tokens: leftTokens + startBufferColumn: @startBufferColumn + ruleStack: @ruleStack + invisibles: @invisibles + lineEnding: null, + indentLevel: @indentLevel, + tabLength: @tabLength + ) + rightFragment = new TokenizedLine( + tokens: indentationTokens.concat(rightTokens) + startBufferColumn: @bufferColumnForScreenColumn(column) + ruleStack: @ruleStack + invisibles: @invisibles + lineEnding: @lineEnding, + indentLevel: @indentLevel, + tabLength: @tabLength + ) [leftFragment, rightFragment] isSoftWrapped: -> @lineEnding is null - isColumnInsideSoftWrapIndentation: (targetColumn) -> - targetColumn < @getSoftWrapIndentationDelta() + isColumnInsideSoftWrapIndentation: (column) -> + return false if @softWrapIndentationTokens.length is 0 - getSoftWrapIndentationDelta: -> - delta = 0 - for tag, index in @tags - if tag >= 0 - if @specialTokens[index] is SoftWrapIndent - delta += tag - else - break - delta + column < @softWrapIndentationDelta + + getSoftWrapIndentationTokens: -> + _.select(@tokens, (token) -> token.isSoftWrapIndentation) + + buildSoftWrapIndentationDelta: -> + _.reduce @softWrapIndentationTokens, ((acc, token) -> acc + token.screenDelta), 0 hasOnlySoftWrapIndentation: -> - @getSoftWrapIndentationDelta() is @text.length + @tokens.length is @softWrapIndentationTokens.length tokenAtBufferColumn: (bufferColumn) -> @tokens[@tokenIndexAtBufferColumn(bufferColumn)] @@ -430,6 +210,58 @@ class TokenizedLine delta = nextDelta delta + breakOutAtomicTokens: (inputTokens) -> + outputTokens = [] + breakOutLeadingSoftTabs = true + column = @startBufferColumn + for token in inputTokens + newTokens = token.breakOutAtomicTokens(@tabLength, breakOutLeadingSoftTabs, column) + column += newToken.value.length for newToken in newTokens + outputTokens.push(newTokens...) + breakOutLeadingSoftTabs = token.isOnlyWhitespace() if breakOutLeadingSoftTabs + outputTokens + + markLeadingAndTrailingWhitespaceTokens: -> + @firstNonWhitespaceIndex = @text.search(NonWhitespaceRegex) + if @firstNonWhitespaceIndex > 0 and isPairedCharacter(@text, @firstNonWhitespaceIndex - 1) + @firstNonWhitespaceIndex-- + firstTrailingWhitespaceIndex = @text.search(TrailingWhitespaceRegex) + @lineIsWhitespaceOnly = firstTrailingWhitespaceIndex is 0 + index = 0 + for token in @tokens + if index < @firstNonWhitespaceIndex + token.firstNonWhitespaceIndex = Math.min(index + token.value.length, @firstNonWhitespaceIndex - index) + # Only the *last* segment of a soft-wrapped line can have trailing whitespace + if @lineEnding? and (index + token.value.length > firstTrailingWhitespaceIndex) + token.firstTrailingWhitespaceIndex = Math.max(0, firstTrailingWhitespaceIndex - index) + index += token.value.length + return + + substituteInvisibleCharacters: -> + invisibles = @invisibles + changedText = false + + for token, i in @tokens + if token.isHardTab + if invisibles.tab + token.value = invisibles.tab + token.value.substring(invisibles.tab.length) + token.hasInvisibleCharacters = true + changedText = true + else + if invisibles.space + if token.hasLeadingWhitespace() and not token.isSoftWrapIndentation + token.value = token.value.replace LeadingWhitespaceRegex, (leadingWhitespace) -> + leadingWhitespace.replace RepeatedSpaceRegex, invisibles.space + token.hasInvisibleCharacters = true + changedText = true + if token.hasTrailingWhitespace() + token.value = token.value.replace TrailingWhitespaceRegex, (leadingWhitespace) -> + leadingWhitespace.replace RepeatedSpaceRegex, invisibles.space + token.hasInvisibleCharacters = true + changedText = true + + @text = @buildText() if changedText + buildEndOfLineInvisibles: -> @endOfLineInvisibles = [] {cr, eol} = @invisibles @@ -442,13 +274,11 @@ class TokenizedLine @endOfLineInvisibles.push(eol) if eol isComment: -> - iterator = @getTokenIterator() - while iterator.next() - scopes = iterator.getScopes() - continue if scopes.length is 1 - continue unless NonWhitespaceRegex.test(iterator.getText()) - for scope in scopes - return true if CommentScopeRegex.test(scope) + for token in @tokens + continue if token.scopes.length is 1 + continue if token.isOnlyWhitespace() + for scope in token.scopes + return true if _.contains(scope.split('.'), 'comment') break false @@ -459,6 +289,42 @@ class TokenizedLine @tokens[index] getTokenCount: -> - count = 0 - count++ for tag in @tags when tag >= 0 - count + @tokens.length + + bufferColumnForToken: (targetToken) -> + column = 0 + for token in @tokens + return column if token is targetToken + column += token.bufferDelta + + getScopeTree: -> + return @scopeTree if @scopeTree? + + scopeStack = [] + for token in @tokens + @updateScopeStack(scopeStack, token.scopes) + _.last(scopeStack).children.push(token) + + @scopeTree = scopeStack[0] + @updateScopeStack(scopeStack, []) + @scopeTree + + updateScopeStack: (scopeStack, desiredScopeDescriptor) -> + # Find a common prefix + for scope, i in desiredScopeDescriptor + break unless scopeStack[i]?.scope is desiredScopeDescriptor[i] + + # Pop scopeDescriptor until we're at the common prefx + until scopeStack.length is i + poppedScope = scopeStack.pop() + _.last(scopeStack)?.children.push(poppedScope) + + # Push onto common prefix until scopeStack equals desiredScopeDescriptor + for j in [i...desiredScopeDescriptor.length] + scopeStack.push(new Scope(desiredScopeDescriptor[j])) + + return + +class Scope + constructor: (@scope) -> + @children = []