From da2df2297a3944070318f139b1c5a3e4ac2d3fc2 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Thu, 14 May 2015 00:11:26 +0200 Subject: [PATCH] Add a TokenIterator and use it for tokens shim --- src/token-iterator.coffee | 85 +++++++++++++++++++++++++++++++++++++++ src/tokenized-line.coffee | 56 +++++++++++++------------- 2 files changed, 113 insertions(+), 28 deletions(-) create mode 100644 src/token-iterator.coffee diff --git a/src/token-iterator.coffee b/src/token-iterator.coffee new file mode 100644 index 000000000..a8b9db59e --- /dev/null +++ b/src/token-iterator.coffee @@ -0,0 +1,85 @@ +{SoftTab, HardTab, PairedCharacter, SoftWrapIndent} = require './special-token-symbols' + +module.exports = +class TokenIterator + @instance: new this + + constructor: (line) -> + @reset(line) if line? + + reset: (@line) -> + @index = null + @bufferStart = 0 + @bufferEnd = 0 + @screenStart = 0 + @screenEnd = 0 + @scopes = @line.openScopes.map (id) -> atom.grammars.scopeForId(id) + @scopeStarts = @scopes.slice() + @scopeEnds = [] + this + + next: -> + {tags} = @line + + if @index? + @index++ + @scopeEnds.length = 0 + @scopeStarts.length = 0 + @bufferStart = @bufferEnd + @screenStart = @screenEnd + else + @index = 0 + + while @index < tags.length + tag = tags[@index] + if tag < 0 + if tag % 2 is 0 + @scopeEnds.push(atom.grammars.scopeForId(tag + 1)) + @scopes.pop() + else + scope = atom.grammars.scopeForId(tag) + @scopeStarts.push(scope) + @scopes.push(scope) + @index++ + else + if @isHardTab() + @screenEnd = @screenStart + tag + @bufferEnd = @bufferStart + 1 + else if @isSoftWrapIndentation() + @screenEnd = @screenStart + tag + @bufferEnd = @bufferStart + 0 + else + @screenEnd = @screenStart + tag + @bufferEnd = @bufferStart + tag + return true + + false + + getBufferStart: -> @bufferStart + getBufferEnd: -> @bufferEnd + + getScreenStart: -> @screenStart + getScreenEnd: -> @screenEnd + + getScopeStarts: -> @scopeStarts + getScopeEnds: -> @scopeEnds + + getScopes: -> @scopes + + getText: -> + @line.text.substring(@screenStart, @screenEnd) + + isSoftTab: -> + @line.specialTokens[@index] is SoftTab + + isHardTab: -> + @line.specialTokens[@index] is HardTab + + isSoftWrapIndentation: -> + @line.specialTokens[@index] is SoftWrapIndent + + isPairedCharacter: -> + @line.specialTokens[@index] is PairedCharacter + + isAtomic: -> + @isSoftTab() or @isHardTab() or @isSoftWrapIndentation() or @isPairedCharacter() diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index cbc56da99..de1c45e96 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -1,6 +1,7 @@ _ = require 'underscore-plus' {isPairedCharacter} = require './text-utils' Token = require './token' +TokenIterator = require './token-iterator' {SoftTab, HardTab, PairedCharacter, SoftWrapIndent} = require './special-token-symbols' NonWhitespaceRegex = /\S/ @@ -28,10 +29,10 @@ class TokenizedLine @startBufferColumn ?= 0 @bufferDelta = @text.length - @subdivideTokens() + @transformContent() @buildEndOfLineInvisibles() if @invisibles? and @lineEnding? - subdivideTokens: -> + transformContent: -> text = '' bufferColumn = 0 screenColumn = 0 @@ -142,37 +143,36 @@ class TokenizedLine @firstTrailingWhitespaceIndex = 0 Object.defineProperty @prototype, 'tokens', get: -> - offset = 0 + iterator = TokenIterator.instance.reset(this) + tokens = [] - atom.grammars.decodeTokens @text, @tags, @openScopes.slice(), (tokenProperties, index) => - switch @specialTokens[index] - when SoftTab - tokenProperties.isAtomic = true - when HardTab - tokenProperties.isAtomic = true - tokenProperties.isHardTab = true - tokenProperties.bufferDelta = 1 - tokenProperties.hasInvisibleCharacters = true if @invisibles?.tab - when PairedCharacter - tokenProperties.isAtomic = true - tokenProperties.hasPairedCharacter = true - when SoftWrapIndent - tokenProperties.isAtomic = true - tokenProperties.isSoftWrapIndentation = true + while iterator.next() + properties = { + value: iterator.getText() + scopes: iterator.getScopes().slice() + isAtomic: iterator.isAtomic() + isHardTab: iterator.isHardTab() + hasPairedCharacter: iterator.isPairedCharacter() + isSoftWrapIndentation: iterator.isSoftWrapIndentation() + } - if offset < @firstNonWhitespaceIndex - tokenProperties.firstNonWhitespaceIndex = - Math.min(tokenProperties.value.length, @firstNonWhitespaceIndex - offset) - tokenProperties.hasInvisibleCharacters = true if @invisibles?.space + if iterator.isHardTab() + properties.bufferDelta = 1 + properties.hasInvisibleCharacters = true if @invisibles?.tab - if @lineEnding? and (offset + tokenProperties.value.length > @firstTrailingWhitespaceIndex) - tokenProperties.firstTrailingWhitespaceIndex = - Math.max(0, @firstTrailingWhitespaceIndex - offset) - tokenProperties.hasInvisibleCharacters = true if @invisibles?.space + if iterator.getScreenStart() < @firstNonWhitespaceIndex + properties.firstNonWhitespaceIndex = + Math.min(@firstNonWhitespaceIndex, iterator.getScreenEnd()) - iterator.getScreenStart() + properties.hasInvisibleCharacters = true if @invisibles?.space - offset += tokenProperties.value.length + if @lineEnding? and iterator.getScreenEnd() > @firstTrailingWhitespaceIndex + properties.firstTrailingWhitespaceIndex = + Math.max(0, @firstTrailingWhitespaceIndex - iterator.getScreenStart()) + properties.hasInvisibleCharacters = true if @invisibles?.space - new Token(tokenProperties) + tokens.push(new Token(properties)) + + tokens copy: -> copy = new TokenizedLine