diff --git a/spec/tokenized-line-spec.coffee b/spec/tokenized-line-spec.coffee index 59270179c..2914ec089 100644 --- a/spec/tokenized-line-spec.coffee +++ b/spec/tokenized-line-spec.coffee @@ -17,24 +17,3 @@ describe "TokenizedLine", -> it "returns false when the line is not only whitespace", -> expect(editor.tokenizedLineForScreenRow(0).isOnlyWhitespace()).toBe false expect(editor.tokenizedLineForScreenRow(2).isOnlyWhitespace()).toBe false - - describe "::getScopeTree()", -> - it "returns a tree whose inner nodes are scopeDescriptor and whose leaf nodes are tokens in those scopeDescriptor", -> - [tokens, tokenIndex] = [] - - ensureValidScopeTree = (scopeTree, scopeDescriptor=[]) -> - if scopeTree.children? - for child in scopeTree.children - ensureValidScopeTree(child, scopeDescriptor.concat([scopeTree.scope])) - else - expect(scopeTree).toEqual tokens[tokenIndex++] - expect(scopeDescriptor).toEqual scopeTree.scopes - - waitsForPromise -> - atom.project.open('coffee.coffee').then (o) -> editor = o - - runs -> - tokenIndex = 0 - tokens = editor.tokenizedLineForScreenRow(1).tokens - scopeTree = editor.tokenizedLineForScreenRow(1).getScopeTree() - ensureValidScopeTree(scopeTree) diff --git a/src/token.coffee b/src/token.coffee index 6a6683a73..60e8194f8 100644 --- a/src/token.coffee +++ b/src/token.coffee @@ -1,13 +1,8 @@ _ = require 'underscore-plus' -textUtils = require './text-utils' -WhitespaceRegexesByTabLength = {} -EscapeRegex = /[&"'<>]/g StartDotRegex = /^\.?/ WhitespaceRegex = /\S/ -MaxTokenLength = 20000 - # Represents a single unit of text as selected by a grammar. module.exports = class Token @@ -28,7 +23,6 @@ class Token @screenDelta = @value.length @bufferDelta ?= @screenDelta - @hasPairedCharacter ?= textUtils.hasPairedCharacter(@value) isEqual: (other) -> # TODO: scopes is deprecated. This is here for the sake of lang package tests @@ -37,126 +31,6 @@ class Token isBracket: -> /^meta\.brace\b/.test(_.last(@scopes)) - splitAt: (splitIndex) -> - leftToken = new Token(value: @value.substring(0, splitIndex), scopes: @scopes) - rightToken = new Token(value: @value.substring(splitIndex), scopes: @scopes) - - if @firstNonWhitespaceIndex? - leftToken.firstNonWhitespaceIndex = Math.min(splitIndex, @firstNonWhitespaceIndex) - leftToken.hasInvisibleCharacters = @hasInvisibleCharacters - - if @firstNonWhitespaceIndex > splitIndex - rightToken.firstNonWhitespaceIndex = @firstNonWhitespaceIndex - splitIndex - rightToken.hasInvisibleCharacters = @hasInvisibleCharacters - - if @firstTrailingWhitespaceIndex? - rightToken.firstTrailingWhitespaceIndex = Math.max(0, @firstTrailingWhitespaceIndex - splitIndex) - rightToken.hasInvisibleCharacters = @hasInvisibleCharacters - - if @firstTrailingWhitespaceIndex < splitIndex - leftToken.firstTrailingWhitespaceIndex = @firstTrailingWhitespaceIndex - leftToken.hasInvisibleCharacters = @hasInvisibleCharacters - - [leftToken, rightToken] - - whitespaceRegexForTabLength: (tabLength) -> - WhitespaceRegexesByTabLength[tabLength] ?= new RegExp("([ ]{#{tabLength}})|(\t)|([^\t]+)", "g") - - breakOutAtomicTokens: (tabLength, breakOutLeadingSoftTabs, startColumn) -> - if @hasPairedCharacter - outputTokens = [] - column = startColumn - - for token in @breakOutPairedCharacters() - if token.isAtomic - outputTokens.push(token) - else - outputTokens.push(token.breakOutAtomicTokens(tabLength, breakOutLeadingSoftTabs, column)...) - breakOutLeadingSoftTabs = token.isOnlyWhitespace() if breakOutLeadingSoftTabs - column += token.value.length - - outputTokens - else - return [this] if @isAtomic - - if breakOutLeadingSoftTabs - return [this] unless /^[ ]|\t/.test(@value) - else - return [this] unless /\t/.test(@value) - - outputTokens = [] - regex = @whitespaceRegexForTabLength(tabLength) - column = startColumn - while match = regex.exec(@value) - [fullMatch, softTab, hardTab] = match - token = null - if softTab and breakOutLeadingSoftTabs - token = @buildSoftTabToken(tabLength) - else if hardTab - breakOutLeadingSoftTabs = false - token = @buildHardTabToken(tabLength, column) - else - breakOutLeadingSoftTabs = false - value = match[0] - token = new Token({value, @scopes}) - column += token.value.length - outputTokens.push(token) - - outputTokens - - breakOutPairedCharacters: -> - outputTokens = [] - index = 0 - nonPairStart = 0 - - while index < @value.length - if textUtils.isPairedCharacter(@value, index) - if nonPairStart isnt index - outputTokens.push(new Token({value: @value[nonPairStart...index], @scopes})) - outputTokens.push(@buildPairedCharacterToken(@value, index)) - index += 2 - nonPairStart = index - else - index++ - - if nonPairStart isnt index - outputTokens.push(new Token({value: @value[nonPairStart...index], @scopes})) - - outputTokens - - buildPairedCharacterToken: (value, index) -> - new Token( - value: value[index..index + 1] - scopes: @scopes - isAtomic: true - hasPairedCharacter: true - ) - - buildHardTabToken: (tabLength, column) -> - @buildTabToken(tabLength, true, column) - - buildSoftTabToken: (tabLength) -> - @buildTabToken(tabLength, false, 0) - - buildTabToken: (tabLength, isHardTab, column=0) -> - tabStop = tabLength - (column % tabLength) - new Token( - value: _.multiplyString(" ", tabStop) - scopes: @scopes - bufferDelta: if isHardTab then 1 else tabStop - isAtomic: true - isHardTab: isHardTab - ) - - buildSoftWrapIndentationToken: (length) -> - new Token( - value: _.multiplyString(" ", length), - scopes: @scopes, - bufferDelta: 0, - isAtomic: true, - isSoftWrapIndentation: true - ) - isOnlyWhitespace: -> not WhitespaceRegex.test(@value) @@ -166,72 +40,6 @@ class Token scopeClasses = scope.split('.') _.isSubset(targetClasses, scopeClasses) - getValueAsHtml: ({hasIndentGuide}) -> - if @isHardTab - classes = 'hard-tab' - classes += ' leading-whitespace' if @hasLeadingWhitespace() - classes += ' trailing-whitespace' if @hasTrailingWhitespace() - classes += ' indent-guide' if hasIndentGuide - classes += ' invisible-character' if @hasInvisibleCharacters - html = "#{@escapeString(@value)}" - else - startIndex = 0 - endIndex = @value.length - - leadingHtml = '' - trailingHtml = '' - - if @hasLeadingWhitespace() - leadingWhitespace = @value.substring(0, @firstNonWhitespaceIndex) - - classes = 'leading-whitespace' - classes += ' indent-guide' if hasIndentGuide - classes += ' invisible-character' if @hasInvisibleCharacters - - leadingHtml = "#{leadingWhitespace}" - startIndex = @firstNonWhitespaceIndex - - if @hasTrailingWhitespace() - tokenIsOnlyWhitespace = @firstTrailingWhitespaceIndex is 0 - trailingWhitespace = @value.substring(@firstTrailingWhitespaceIndex) - - classes = 'trailing-whitespace' - classes += ' indent-guide' if hasIndentGuide and not @hasLeadingWhitespace() and tokenIsOnlyWhitespace - classes += ' invisible-character' if @hasInvisibleCharacters - - trailingHtml = "#{trailingWhitespace}" - - endIndex = @firstTrailingWhitespaceIndex - - html = leadingHtml - if @value.length > MaxTokenLength - while startIndex < endIndex - html += "" + @escapeString(@value, startIndex, startIndex + MaxTokenLength) + "" - startIndex += MaxTokenLength - else - html += @escapeString(@value, startIndex, endIndex) - - html += trailingHtml - html - - escapeString: (str, startIndex, endIndex) -> - strLength = str.length - - startIndex ?= 0 - endIndex ?= strLength - - str = str.slice(startIndex, endIndex) if startIndex > 0 or endIndex < strLength - str.replace(EscapeRegex, @escapeStringReplace) - - escapeStringReplace: (match) -> - switch match - when '&' then '&' - when '"' then '"' - when "'" then ''' - when '<' then '<' - when '>' then '>' - else match - hasLeadingWhitespace: -> @firstNonWhitespaceIndex? and @firstNonWhitespaceIndex > 0 diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index c10c6693d..99db15c3b 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -488,35 +488,3 @@ class TokenizedLine getTokenCount: -> @tokens.length - - getScopeTree: -> - return @scopeTree if @scopeTree? - - scopeStack = [] - for token in @tokens - @updateScopeStack(scopeStack, token.scopes) - _.last(scopeStack).children.push(token) - - @scopeTree = scopeStack[0] - @updateScopeStack(scopeStack, []) - @scopeTree - - updateScopeStack: (scopeStack, desiredScopeDescriptor) -> - # Find a common prefix - for scope, i in desiredScopeDescriptor - break unless scopeStack[i]?.scope is desiredScopeDescriptor[i] - - # Pop scopeDescriptor until we're at the common prefx - until scopeStack.length is i - poppedScope = scopeStack.pop() - _.last(scopeStack)?.children.push(poppedScope) - - # Push onto common prefix until scopeStack equals desiredScopeDescriptor - for j in [i...desiredScopeDescriptor.length] - scopeStack.push(new Scope(desiredScopeDescriptor[j])) - - return - -class Scope - constructor: (@scope) -> - @children = []