From 9583ff04e6a6c711273c2decda2a352321565f16 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Wed, 6 May 2015 22:03:10 +0200 Subject: [PATCH 01/50] WIP: Start integrating with nested words version of first-mate --- src/tokenized-buffer.coffee | 51 ++++++++++++++++++++++++++----------- src/tokenized-line.coffee | 9 +++++-- 2 files changed, 43 insertions(+), 17 deletions(-) diff --git a/src/tokenized-buffer.coffee b/src/tokenized-buffer.coffee index d9f1bcba7..082c923dd 100644 --- a/src/tokenized-buffer.coffee +++ b/src/tokenized-buffer.coffee @@ -153,7 +153,7 @@ class TokenizedBuffer extends Model row = startRow loop previousStack = @stackForRow(row) - @tokenizedLines[row] = @buildTokenizedLineForRow(row, @stackForRow(row - 1)) + @tokenizedLines[row] = @buildTokenizedLineForRow(row, @stackForRow(row - 1), @parentScopesForRow(row)) if --rowsRemaining is 0 filledRegion = false endRow = row @@ -213,7 +213,7 @@ class TokenizedBuffer extends Model @updateInvalidRows(start, end, delta) previousEndStack = @stackForRow(end) # used in spill detection below - newTokenizedLines = @buildTokenizedLinesForRows(start, end + delta, @stackForRow(start - 1)) + newTokenizedLines = @buildTokenizedLinesForRows(start, end + delta, @stackForRow(start - 1), @parentScopesForRow(start)) _.spliceWithArray(@tokenizedLines, start, end - start + 1, newTokenizedLines) start = @retokenizeWhitespaceRowsIfIndentLevelChanged(start - 1, -1) @@ -234,7 +234,7 @@ class TokenizedBuffer extends Model line = @tokenizedLines[row] if line?.isOnlyWhitespace() and @indentLevelForRow(row) isnt line.indentLevel while line?.isOnlyWhitespace() - @tokenizedLines[row] = @buildTokenizedLineForRow(row, @stackForRow(row - 1)) + @tokenizedLines[row] = @buildTokenizedLineForRow(row, @stackForRow(row - 1), @parentScopesForRow(row)) row += increment line = @tokenizedLines[row] @@ -276,16 +276,18 @@ class TokenizedBuffer extends Model @tokenizedLineForRow(row).isComment() and @tokenizedLineForRow(nextRow).isComment() - buildTokenizedLinesForRows: (startRow, endRow, startingStack) -> + buildTokenizedLinesForRows: (startRow, endRow, startingStack, startingParentScopes) -> ruleStack = startingStack + parentScopes = startingParentScopes stopTokenizingAt = startRow + @chunkSize tokenizedLines = for row in [startRow..endRow] if (ruleStack or row is 0) and row < stopTokenizingAt - screenLine = @buildTokenizedLineForRow(row, ruleStack) - ruleStack = screenLine.ruleStack + tokenizedLine = @buildTokenizedLineForRow(row, ruleStack, parentScopes) + ruleStack = tokenizedLine.ruleStack + parentScopes = @scopesFromContent(parentScopes, tokenizedLine.content) else - screenLine = @buildPlaceholderTokenizedLineForRow(row) - screenLine + tokenizedLine = @buildPlaceholderTokenizedLineForRow(row, parentScopes) + tokenizedLine if endRow >= stopTokenizingAt @invalidateRow(stopTokenizingAt) @@ -298,21 +300,22 @@ class TokenizedBuffer extends Model buildPlaceholderTokenizedLineForRow: (row) -> line = @buffer.lineForRow(row) - tokens = [new Token(value: line, scopes: [@grammar.scopeName])] + parentScopes = [@grammar.idForScope(@grammar.scopeName)] + content = [line] tabLength = @getTabLength() indentLevel = @indentLevelForRow(row) lineEnding = @buffer.lineEndingForRow(row) - new TokenizedLine({tokens, tabLength, indentLevel, @invisibles, lineEnding}) + new TokenizedLine({parentScopes, content, tabLength, indentLevel, @invisibles, lineEnding}) - buildTokenizedLineForRow: (row, ruleStack) -> - @buildTokenizedLineForRowWithText(row, @buffer.lineForRow(row), ruleStack) + buildTokenizedLineForRow: (row, ruleStack, parentScopes) -> + @buildTokenizedLineForRowWithText(row, @buffer.lineForRow(row), ruleStack, parentScopes) - buildTokenizedLineForRowWithText: (row, line, ruleStack = @stackForRow(row - 1)) -> + buildTokenizedLineForRowWithText: (row, line, ruleStack = @stackForRow(row - 1), parentScopes = @parentScopesForRow(row)) -> lineEnding = @buffer.lineEndingForRow(row) tabLength = @getTabLength() indentLevel = @indentLevelForRow(row) - {tokens, ruleStack} = @grammar.tokenizeLine(line, ruleStack, row is 0) - new TokenizedLine({tokens, ruleStack, tabLength, lineEnding, indentLevel, @invisibles}) + {content, ruleStack} = @grammar.tokenizeLine(line, ruleStack, row is 0) + new TokenizedLine({parentScopes, content, ruleStack, tabLength, lineEnding, indentLevel, @invisibles}) tokenizedLineForRow: (bufferRow) -> @tokenizedLines[bufferRow] @@ -320,6 +323,24 @@ class TokenizedBuffer extends Model stackForRow: (bufferRow) -> @tokenizedLines[bufferRow]?.ruleStack + parentScopesForRow: (bufferRow) -> + if bufferRow > 0 + precedingLine = @tokenizedLines[bufferRow - 1] + @scopesFromContent(precedingLine.parentScopes, precedingLine.content) + else + [] + + scopesFromContent: (startingScopes, content) -> + scopes = startingScopes.slice() + for symbol in content when typeof symbol is 'number' + if symbol > 0 + scopes.push(symbol) + else + popped = scopes.pop() + unless -popped is symbol + throw new Error("Encountered an invalid scope end id. Popped #{popped}, expected to pop #{-symbol}.") + scopes + indentLevelForRow: (bufferRow) -> line = @buffer.lineForRow(bufferRow) indentLevel = 0 diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index b81d972a0..942d72c1a 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -1,5 +1,6 @@ _ = require 'underscore-plus' {isPairedCharacter} = require './text-utils' +Token = require './token' NonWhitespaceRegex = /\S/ LeadingWhitespaceRegex = /^\s*/ @@ -14,9 +15,9 @@ class TokenizedLine firstNonWhitespaceIndex: 0 foldable: false - constructor: ({tokens, @lineEnding, @ruleStack, @startBufferColumn, @fold, @tabLength, @indentLevel, @invisibles}) -> + constructor: ({@parentScopes, @content, @lineEnding, @ruleStack, @startBufferColumn, @fold, @tabLength, @indentLevel, @invisibles}) -> @startBufferColumn ?= 0 - @tokens = @breakOutAtomicTokens(tokens) + # @tokens = @breakOutAtomicTokens(tokens) @text = @buildText() @bufferDelta = @buildBufferDelta() @softWrapIndentationTokens = @getSoftWrapIndentationTokens() @@ -28,6 +29,10 @@ class TokenizedLine @substituteInvisibleCharacters() @buildEndOfLineInvisibles() if @lineEnding? + Object.defineProperty @prototype, 'tokens', get: -> + tokens = atom.grammars.decodeContent(@parentScopes.concat(@content)) + tokens.map (properties) -> new Token(properties) + buildText: -> text = "" text += token.value for token in @tokens From 06b5d273575645844c76391ed6fb8cb55903c7d7 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Fri, 8 May 2015 20:57:46 +0200 Subject: [PATCH 02/50] Adapt to first-mate returning purely numeric tag arrays --- src/tokenized-buffer.coffee | 31 ++++++++++++++++--------------- src/tokenized-line.coffee | 9 +++++---- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/src/tokenized-buffer.coffee b/src/tokenized-buffer.coffee index 082c923dd..9dde39afe 100644 --- a/src/tokenized-buffer.coffee +++ b/src/tokenized-buffer.coffee @@ -284,7 +284,7 @@ class TokenizedBuffer extends Model if (ruleStack or row is 0) and row < stopTokenizingAt tokenizedLine = @buildTokenizedLineForRow(row, ruleStack, parentScopes) ruleStack = tokenizedLine.ruleStack - parentScopes = @scopesFromContent(parentScopes, tokenizedLine.content) + parentScopes = @scopesFromTags(parentScopes, tokenizedLine.tags) else tokenizedLine = @buildPlaceholderTokenizedLineForRow(row, parentScopes) tokenizedLine @@ -299,23 +299,23 @@ class TokenizedBuffer extends Model @buildPlaceholderTokenizedLineForRow(row) for row in [startRow..endRow] buildPlaceholderTokenizedLineForRow: (row) -> - line = @buffer.lineForRow(row) parentScopes = [@grammar.idForScope(@grammar.scopeName)] - content = [line] + text = @buffer.lineForRow(row) + tags = [text.length] tabLength = @getTabLength() indentLevel = @indentLevelForRow(row) lineEnding = @buffer.lineEndingForRow(row) - new TokenizedLine({parentScopes, content, tabLength, indentLevel, @invisibles, lineEnding}) + new TokenizedLine({parentScopes, text, tags, tabLength, indentLevel, @invisibles, lineEnding}) buildTokenizedLineForRow: (row, ruleStack, parentScopes) -> @buildTokenizedLineForRowWithText(row, @buffer.lineForRow(row), ruleStack, parentScopes) - buildTokenizedLineForRowWithText: (row, line, ruleStack = @stackForRow(row - 1), parentScopes = @parentScopesForRow(row)) -> + buildTokenizedLineForRowWithText: (row, text, ruleStack = @stackForRow(row - 1), parentScopes = @parentScopesForRow(row)) -> lineEnding = @buffer.lineEndingForRow(row) tabLength = @getTabLength() indentLevel = @indentLevelForRow(row) - {content, ruleStack} = @grammar.tokenizeLine(line, ruleStack, row is 0) - new TokenizedLine({parentScopes, content, ruleStack, tabLength, lineEnding, indentLevel, @invisibles}) + {tags, ruleStack} = @grammar.tokenizeLine(text, ruleStack, row is 0) + new TokenizedLine({parentScopes, text, tags, ruleStack, tabLength, lineEnding, indentLevel, @invisibles}) tokenizedLineForRow: (bufferRow) -> @tokenizedLines[bufferRow] @@ -326,19 +326,20 @@ class TokenizedBuffer extends Model parentScopesForRow: (bufferRow) -> if bufferRow > 0 precedingLine = @tokenizedLines[bufferRow - 1] - @scopesFromContent(precedingLine.parentScopes, precedingLine.content) + @scopesFromTags(precedingLine.parentScopes, precedingLine.tags) else [] - scopesFromContent: (startingScopes, content) -> + scopesFromTags: (startingScopes, tags) -> scopes = startingScopes.slice() - for symbol in content when typeof symbol is 'number' - if symbol > 0 - scopes.push(symbol) + for tag in tags when tag < 0 + if (tag % 2) is -1 + scopes.push(tag) else - popped = scopes.pop() - unless -popped is symbol - throw new Error("Encountered an invalid scope end id. Popped #{popped}, expected to pop #{-symbol}.") + expectedScope = tag + 1 + poppedScope = scopes.pop() + unless poppedScope is expectedScope + throw new Error("Encountered an invalid scope end id. Popped #{poppedScope}, expected to pop #{expectedScope}.") scopes indentLevelForRow: (bufferRow) -> diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index 942d72c1a..be8a16799 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -15,10 +15,9 @@ class TokenizedLine firstNonWhitespaceIndex: 0 foldable: false - constructor: ({@parentScopes, @content, @lineEnding, @ruleStack, @startBufferColumn, @fold, @tabLength, @indentLevel, @invisibles}) -> + constructor: ({@parentScopes, @text, @tags, @lineEnding, @ruleStack, @startBufferColumn, @fold, @tabLength, @indentLevel, @invisibles}) -> @startBufferColumn ?= 0 # @tokens = @breakOutAtomicTokens(tokens) - @text = @buildText() @bufferDelta = @buildBufferDelta() @softWrapIndentationTokens = @getSoftWrapIndentationTokens() @softWrapIndentationDelta = @buildSoftWrapIndentationDelta() @@ -30,8 +29,10 @@ class TokenizedLine @buildEndOfLineInvisibles() if @lineEnding? Object.defineProperty @prototype, 'tokens', get: -> - tokens = atom.grammars.decodeContent(@parentScopes.concat(@content)) - tokens.map (properties) -> new Token(properties) + tokens = atom.grammars.decodeContent(@text, @tags, @parentScopes.slice()) + tokens.map (properties, index) => + properties.isAtomic = true if @specialTokens[index] is SoftTab + new Token(properties) buildText: -> text = "" From 43806d64167c83defd86fa51b359f3a841feaac1 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Fri, 8 May 2015 20:57:57 +0200 Subject: [PATCH 03/50] Break out leading soft tabs after switching to numeric tag arrays --- src/tokenized-line.coffee | 41 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index be8a16799..e99ceba8f 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -2,6 +2,8 @@ _ = require 'underscore-plus' {isPairedCharacter} = require './text-utils' Token = require './token' +SoftTab = Symbol('SoftTab') + NonWhitespaceRegex = /\S/ LeadingWhitespaceRegex = /^\s*/ TrailingWhitespaceRegex = /\s*$/ @@ -17,6 +19,11 @@ class TokenizedLine constructor: ({@parentScopes, @text, @tags, @lineEnding, @ruleStack, @startBufferColumn, @fold, @tabLength, @indentLevel, @invisibles}) -> @startBufferColumn ?= 0 + + @specialTokens = {} + + @subdivideTokens() + # @tokens = @breakOutAtomicTokens(tokens) @bufferDelta = @buildBufferDelta() @softWrapIndentationTokens = @getSoftWrapIndentationTokens() @@ -28,6 +35,40 @@ class TokenizedLine @substituteInvisibleCharacters() @buildEndOfLineInvisibles() if @lineEnding? + subdivideTokens: -> + text = '' + bufferColumn = 0 + screenColumn = 0 + tokenIndex = 0 + tokenOffset = 0 + inLeadingWhitespace = true + + while bufferColumn < @text.length + # advance to next token if we've iterated over its length + if tokenOffset is @tags[tokenIndex] + tokenIndex++ + tokenOffset = 0 + + # advance to next token tag + tokenIndex++ while @tags[tokenIndex] < 0 + + character = @text[bufferColumn] + + # split out leading soft tabs + if character is ' ' + if inLeadingWhitespace and (screenColumn + 1) % @tabLength is 0 + @specialTokens[tokenIndex] = SoftTab + @tags.splice(tokenIndex, 1, @tabLength, @tags[tokenIndex] - @tabLength) + else + inLeadingWhitespace = false + + text += character + bufferColumn++ + screenColumn++ + tokenOffset++ + + @text = text + Object.defineProperty @prototype, 'tokens', get: -> tokens = atom.grammars.decodeContent(@text, @tags, @parentScopes.slice()) tokens.map (properties, index) => From 9554bfd3938307a240ecc18b312eb47efefa6c01 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Sat, 9 May 2015 01:07:50 +0200 Subject: [PATCH 04/50] Break out hard tabs and paired characters from numeric tag arrays --- src/tokenized-line.coffee | 82 +++++++++++++++++++++++++++++++++------ 1 file changed, 71 insertions(+), 11 deletions(-) diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index e99ceba8f..6f86847de 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -3,6 +3,8 @@ _ = require 'underscore-plus' Token = require './token' SoftTab = Symbol('SoftTab') +HardTab = Symbol('HardTab') +PairedCharacter = Symbol('PairedCharacter') NonWhitespaceRegex = /\S/ LeadingWhitespaceRegex = /^\s*/ @@ -54,26 +56,84 @@ class TokenizedLine character = @text[bufferColumn] + # split out unicode surrogate pairs + if isPairedCharacter(@text, bufferColumn) + prefix = tokenOffset + suffix = @tags[tokenIndex] - tokenOffset - 2 + splitTokens = [] + splitTokens.push(prefix) if prefix > 0 + splitTokens.push(2) + splitTokens.push(suffix) if suffix > 0 + + @tags.splice(tokenIndex, 1, splitTokens...) + + text += @text.substr(bufferColumn, 2) + screenColumn++ + bufferColumn += 2 + + tokenIndex++ if prefix > 0 + @specialTokens[tokenIndex] = PairedCharacter + tokenIndex++ + tokenOffset = 0 + # split out leading soft tabs - if character is ' ' + else if character is ' ' and inLeadingWhitespace if inLeadingWhitespace and (screenColumn + 1) % @tabLength is 0 @specialTokens[tokenIndex] = SoftTab - @tags.splice(tokenIndex, 1, @tabLength, @tags[tokenIndex] - @tabLength) + suffix = @tags[tokenIndex] - @tabLength + @tags.splice(tokenIndex, 1, @tabLength) + @tags.splice(tokenIndex + 1, 0, suffix) if suffix > 0 + + text += character + screenColumn++ + bufferColumn++ + tokenOffset++ + + # expand hard tabs to the next tab stop + else if character is '\t' + tabLength = @tabLength - (screenColumn % @tabLength) + text += ' ' for i in [0...tabLength] by 1 + + prefix = tokenOffset + suffix = @tags[tokenIndex] - tokenOffset - 1 + splitTokens = [] + splitTokens.push(prefix) if prefix > 0 + splitTokens.push(tabLength) + splitTokens.push(suffix) if suffix > 0 + + @tags.splice(tokenIndex, 1, splitTokens...) + + screenColumn += tabLength + bufferColumn++ + + tokenIndex++ if prefix > 0 + @specialTokens[tokenIndex] = HardTab + tokenIndex++ + tokenOffset = 0 + + # continue past any other character else inLeadingWhitespace = false - - text += character - bufferColumn++ - screenColumn++ - tokenOffset++ + text += character + screenColumn++ + bufferColumn++ + tokenOffset++ @text = text Object.defineProperty @prototype, 'tokens', get: -> - tokens = atom.grammars.decodeContent(@text, @tags, @parentScopes.slice()) - tokens.map (properties, index) => - properties.isAtomic = true if @specialTokens[index] is SoftTab - new Token(properties) + atom.grammars.decodeContent @text, @tags, @parentScopes.slice(), (tokenProperties, index) => + switch @specialTokens[index] + when SoftTab + tokenProperties.isAtomic = true + when HardTab + tokenProperties.isAtomic = true + tokenProperties.bufferDelta = 1 + when PairedCharacter + tokenProperties.isAtomic = true + tokenProperties.hasPairedCharacter = true + + new Token(tokenProperties) buildText: -> text = "" From 9d93d18a8f85d7cfbd5692d6e2799abb7b3c2624 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Sun, 10 May 2015 09:55:23 +0200 Subject: [PATCH 05/50] Mark leading/trailing whitespace in new TokenizedLine representation --- src/token.coffee | 6 +++++- src/tokenized-line.coffee | 25 ++++++++++++++----------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/token.coffee b/src/token.coffee index 8aa4a8706..583e25e9b 100644 --- a/src/token.coffee +++ b/src/token.coffee @@ -20,7 +20,11 @@ class Token firstTrailingWhitespaceIndex: null hasInvisibleCharacters: false - constructor: ({@value, @scopes, @isAtomic, @bufferDelta, @isHardTab, @hasPairedCharacter, @isSoftWrapIndentation}) -> + constructor: (properties) -> + {@value, @scopes, @isAtomic, @bufferDelta, @isHardTab, @hasPairedCharacter, @isSoftWrapIndentation} = properties + @firstNonWhitespaceIndex = properties.firstNonWhitespaceIndex ? null + @firstTrailingWhitespaceIndex = properties.firstTrailingWhitespaceIndex ? null + @screenDelta = @value.length @bufferDelta ?= @screenDelta @hasPairedCharacter ?= textUtils.hasPairedCharacter(@value) diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index 6f86847de..29ed6b88a 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -122,6 +122,8 @@ class TokenizedLine @text = text Object.defineProperty @prototype, 'tokens', get: -> + offset = 0 + atom.grammars.decodeContent @text, @tags, @parentScopes.slice(), (tokenProperties, index) => switch @specialTokens[index] when SoftTab @@ -133,6 +135,16 @@ class TokenizedLine tokenProperties.isAtomic = true tokenProperties.hasPairedCharacter = true + if offset < @firstNonWhitespaceIndex + tokenProperties.firstNonWhitespaceIndex = + Math.min(offset + tokenProperties.value.length, @firstNonWhitespaceIndex - offset) + + if @lineEnding? and (offset + tokenProperties.value.length > @firstTrailingWhitespaceIndex) + tokenProperties.firstTrailingWhitespaceIndex = + Math.max(0, @firstTrailingWhitespaceIndex - offset) + + offset += tokenProperties.value.length + new Token(tokenProperties) buildText: -> @@ -332,17 +344,8 @@ class TokenizedLine @firstNonWhitespaceIndex = @text.search(NonWhitespaceRegex) if @firstNonWhitespaceIndex > 0 and isPairedCharacter(@text, @firstNonWhitespaceIndex - 1) @firstNonWhitespaceIndex-- - firstTrailingWhitespaceIndex = @text.search(TrailingWhitespaceRegex) - @lineIsWhitespaceOnly = firstTrailingWhitespaceIndex is 0 - index = 0 - for token in @tokens - if index < @firstNonWhitespaceIndex - token.firstNonWhitespaceIndex = Math.min(index + token.value.length, @firstNonWhitespaceIndex - index) - # Only the *last* segment of a soft-wrapped line can have trailing whitespace - if @lineEnding? and (index + token.value.length > firstTrailingWhitespaceIndex) - token.firstTrailingWhitespaceIndex = Math.max(0, firstTrailingWhitespaceIndex - index) - index += token.value.length - return + @firstTrailingWhitespaceIndex = @text.search(TrailingWhitespaceRegex) + @lineIsWhitespaceOnly = @firstTrailingWhitespaceIndex is 0 substituteInvisibleCharacters: -> invisibles = @invisibles From f3f609861ea7dad88f61fd377ca7f9f0efed8942 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Sun, 10 May 2015 10:21:00 +0200 Subject: [PATCH 06/50] Make TokenizedLine::softWrapAt work with new token representation --- src/tokenized-line.coffee | 47 ++++++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index 29ed6b88a..f8204ee51 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -259,23 +259,38 @@ class TokenizedLine indentTokens softWrapAt: (column, hangingIndent) -> - return [new TokenizedLine([], '', [0, 0], [0, 0]), this] if column is 0 + return [null, this] if column is 0 - rightTokens = new Array(@tokens...) - leftTokens = [] - leftScreenColumn = 0 + leftText = @text.substring(0, column) + rightText = @text.substring(column) - while leftScreenColumn < column - if leftScreenColumn + rightTokens[0].screenDelta > column - rightTokens[0..0] = rightTokens[0].splitAt(column - leftScreenColumn) - nextToken = rightTokens.shift() - leftScreenColumn += nextToken.screenDelta - leftTokens.push nextToken + leftTags = [] + rightParentScopes = @parentScopes.slice() - indentationTokens = @buildSoftWrapIndentationTokens(leftTokens[0], hangingIndent) + screenColumn = 0 + for tag, index in @tags + if tag >= 0 + if screenColumn + tag < column + screenColumn += tag + else + leftTags.push(column - screenColumn) + rightTags = @tags.slice(index + 1) + rightPrefix = screenColumn + tag - column + rightTags.unshift(rightPrefix) if rightPrefix > 0 + break + else if (tag % 2) is -1 + rightParentScopes.push(tag) + else + rightParentScopes.pop() + leftTags.push(tag) + + softWrapIndent = @indentLevel * @tabLength + (hangingIndent ? 0) + rightTags.unshift(softWrapIndent) if softWrapIndent > 0 leftFragment = new TokenizedLine( - tokens: leftTokens + parentScopes: @parentScopes + text: leftText + tags: leftTags startBufferColumn: @startBufferColumn ruleStack: @ruleStack invisibles: @invisibles @@ -284,12 +299,14 @@ class TokenizedLine tabLength: @tabLength ) rightFragment = new TokenizedLine( - tokens: indentationTokens.concat(rightTokens) + parentScopes: rightParentScopes + text: rightText + tags: rightTags startBufferColumn: @bufferColumnForScreenColumn(column) ruleStack: @ruleStack invisibles: @invisibles - lineEnding: @lineEnding, - indentLevel: @indentLevel, + lineEnding: @lineEnding + indentLevel: @indentLevel tabLength: @tabLength ) [leftFragment, rightFragment] From 6274ac25fab2efe4ae28e63688720f4e62551315 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Sun, 10 May 2015 10:21:11 +0200 Subject: [PATCH 07/50] Fix TokenizedLine::copy for new representation --- src/tokenized-line.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index f8204ee51..a990c3fe3 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -158,7 +158,7 @@ class TokenizedLine delta copy: -> - new TokenizedLine({@tokens, @lineEnding, @ruleStack, @startBufferColumn, @fold}) + new TokenizedLine({@parentScopes, @text, @tags, @lineEnding, @ruleStack, @startBufferColumn, @fold}) # This clips a given screen column to a valid column that's within the line # and not in the middle of any atomic tokens. From 6eb61d977d0c5d166580fcc5ca3d841f5ab28b0b Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Sun, 10 May 2015 10:23:01 +0200 Subject: [PATCH 08/50] =?UTF-8?q?Delete=20spec=20that=E2=80=99s=20no=20lon?= =?UTF-8?q?ger=20relevant?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- spec/tokenized-buffer-spec.coffee | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/spec/tokenized-buffer-spec.coffee b/spec/tokenized-buffer-spec.coffee index 3cd776c2b..768e77ba4 100644 --- a/spec/tokenized-buffer-spec.coffee +++ b/spec/tokenized-buffer-spec.coffee @@ -695,21 +695,6 @@ describe "TokenizedBuffer", -> expect(line.tokens[0].firstNonWhitespaceIndex).toBe 2 expect(line.tokens[line.tokens.length - 1].firstTrailingWhitespaceIndex).toBe 0 - it "sets the ::firstNonWhitespaceIndex and ::firstTrailingWhitespaceIndex correctly when tokens are split for soft-wrapping", -> - tokenizedBuffer.setInvisibles(space: 'S') - buffer.setText(" token ") - fullyTokenize(tokenizedBuffer) - token = tokenizedBuffer.tokenizedLines[0].tokens[0] - - [leftToken, rightToken] = token.splitAt(1) - expect(leftToken.hasInvisibleCharacters).toBe true - expect(leftToken.firstNonWhitespaceIndex).toBe 1 - expect(leftToken.firstTrailingWhitespaceIndex).toBe null - - expect(leftToken.hasInvisibleCharacters).toBe true - expect(rightToken.firstNonWhitespaceIndex).toBe null - expect(rightToken.firstTrailingWhitespaceIndex).toBe 5 - describe ".indentLevel on tokenized lines", -> beforeEach -> buffer = atom.project.bufferForPathSync('sample.js') From 24967afed14a2ca829991d1f5480b28a3e22c82d Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Mon, 11 May 2015 20:20:30 +0200 Subject: [PATCH 09/50] Substitute invisibles in initial scan --- src/tokenized-line.coffee | 99 +++++++++++++++++++++++++++------------ 1 file changed, 70 insertions(+), 29 deletions(-) diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index a990c3fe3..063d2f6da 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -19,23 +19,23 @@ class TokenizedLine firstNonWhitespaceIndex: 0 foldable: false - constructor: ({@parentScopes, @text, @tags, @lineEnding, @ruleStack, @startBufferColumn, @fold, @tabLength, @indentLevel, @invisibles}) -> - @startBufferColumn ?= 0 - + constructor: (properties) -> + @id = idCounter++ @specialTokens = {} + return unless properties? + + {@parentScopes, @text, @tags, @lineEnding, @ruleStack} = properties + {@startBufferColumn, @fold, @tabLength, @indentLevel, @invisibles} = properties + + @startBufferColumn ?= 0 + @bufferDelta = @text.length + @subdivideTokens() + @buildEndOfLineInvisibles() if @invisibles? and @lineEnding? - # @tokens = @breakOutAtomicTokens(tokens) - @bufferDelta = @buildBufferDelta() - @softWrapIndentationTokens = @getSoftWrapIndentationTokens() - @softWrapIndentationDelta = @buildSoftWrapIndentationDelta() - - @id = idCounter++ - @markLeadingAndTrailingWhitespaceTokens() - if @invisibles - @substituteInvisibleCharacters() - @buildEndOfLineInvisibles() if @lineEnding? + # @softWrapIndentationTokens = @getSoftWrapIndentationTokens() + # @softWrapIndentationDelta = @buildSoftWrapIndentationDelta() subdivideTokens: -> text = '' @@ -43,7 +43,8 @@ class TokenizedLine screenColumn = 0 tokenIndex = 0 tokenOffset = 0 - inLeadingWhitespace = true + firstNonWhitespaceColumn = null + lastNonWhitespaceColumn = null while bufferColumn < @text.length # advance to next token if we've iterated over its length @@ -67,6 +68,9 @@ class TokenizedLine @tags.splice(tokenIndex, 1, splitTokens...) + firstNonWhitespaceColumn ?= screenColumn + lastNonWhitespaceColumn = screenColumn + text += @text.substr(bufferColumn, 2) screenColumn++ bufferColumn += 2 @@ -77,14 +81,17 @@ class TokenizedLine tokenOffset = 0 # split out leading soft tabs - else if character is ' ' and inLeadingWhitespace - if inLeadingWhitespace and (screenColumn + 1) % @tabLength is 0 - @specialTokens[tokenIndex] = SoftTab - suffix = @tags[tokenIndex] - @tabLength - @tags.splice(tokenIndex, 1, @tabLength) - @tags.splice(tokenIndex + 1, 0, suffix) if suffix > 0 + else if character is ' ' + if firstNonWhitespaceColumn? + text += ' ' + else + if (screenColumn + 1) % @tabLength is 0 + @specialTokens[tokenIndex] = SoftTab + suffix = @tags[tokenIndex] - @tabLength + @tags.splice(tokenIndex, 1, @tabLength) + @tags.splice(tokenIndex + 1, 0, suffix) if suffix > 0 + text += @invisibles?.space ? ' ' - text += character screenColumn++ bufferColumn++ tokenOffset++ @@ -92,7 +99,11 @@ class TokenizedLine # expand hard tabs to the next tab stop else if character is '\t' tabLength = @tabLength - (screenColumn % @tabLength) - text += ' ' for i in [0...tabLength] by 1 + if @invisibles?.tab + text += @invisibles.tab + else + text += ' ' + text += ' ' for i in [1...tabLength] by 1 prefix = tokenOffset suffix = @tags[tokenIndex] - tokenOffset - 1 @@ -113,7 +124,9 @@ class TokenizedLine # continue past any other character else - inLeadingWhitespace = false + firstNonWhitespaceColumn ?= screenColumn + lastNonWhitespaceColumn = screenColumn + text += character screenColumn++ bufferColumn++ @@ -121,6 +134,19 @@ class TokenizedLine @text = text + @firstNonWhitespaceIndex = firstNonWhitespaceColumn + if lastNonWhitespaceColumn? + if lastNonWhitespaceColumn + 1 < @text.length + @firstTrailingWhitespaceIndex = lastNonWhitespaceColumn + 1 + if @invisibles?.space + @text = + @text.substring(0, @firstTrailingWhitespaceIndex) + + @text.substring(@firstTrailingWhitespaceIndex) + .replace(RepeatedSpaceRegex, @invisibles.space) + else + @lineIsWhitespaceOnly = true + @firstTrailingWhitespaceIndex = 0 + Object.defineProperty @prototype, 'tokens', get: -> offset = 0 @@ -137,7 +163,7 @@ class TokenizedLine if offset < @firstNonWhitespaceIndex tokenProperties.firstNonWhitespaceIndex = - Math.min(offset + tokenProperties.value.length, @firstNonWhitespaceIndex - offset) + Math.min(tokenProperties.value.length, @firstNonWhitespaceIndex - offset) if @lineEnding? and (offset + tokenProperties.value.length > @firstTrailingWhitespaceIndex) tokenProperties.firstTrailingWhitespaceIndex = @@ -158,7 +184,20 @@ class TokenizedLine delta copy: -> - new TokenizedLine({@parentScopes, @text, @tags, @lineEnding, @ruleStack, @startBufferColumn, @fold}) + copy = new TokenizedLine + copy.indentLevel = @indentLevel + copy.parentScopes = @parentScopes + copy.text = @text + copy.tags = @tags + copy.specialTokens = @specialTokens + copy.firstNonWhitespaceIndex = @firstNonWhitespaceIndex + copy.firstTrailingWhitespaceIndex = @firstTrailingWhitespaceIndex + copy.lineEnding = @lineEnding + copy.endOfLineInvisibles = @endOfLineInvisibles + copy.ruleStack = @ruleStack + copy.startBufferColumn = @startBufferColumn + copy.fold = @fold + copy # This clips a given screen column to a valid column that's within the line # and not in the middle of any atomic tokens. @@ -315,12 +354,14 @@ class TokenizedLine @lineEnding is null isColumnInsideSoftWrapIndentation: (column) -> - return false if @softWrapIndentationTokens.length is 0 - - column < @softWrapIndentationDelta + false + # return false if @softWrapIndentationTokens.length is 0 + # + # column < @softWrapIndentationDelta getSoftWrapIndentationTokens: -> - _.select(@tokens, (token) -> token.isSoftWrapIndentation) + [] + # _.select(@tokens, (token) -> token.isSoftWrapIndentation) buildSoftWrapIndentationDelta: -> _.reduce @softWrapIndentationTokens, ((acc, token) -> acc + token.screenDelta), 0 From a8d01bcec1a8740966f5ee011fc33b05f9f0741c Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Mon, 11 May 2015 23:35:52 +0200 Subject: [PATCH 10/50] Fix bufferRangeForScopeAtPosition with new tags array scheme --- spec/tokenized-buffer-spec.coffee | 4 +-- src/token.coffee | 2 +- src/tokenized-buffer.coffee | 57 +++++++++++++++++++++++-------- src/tokenized-line.coffee | 6 ---- 4 files changed, 46 insertions(+), 23 deletions(-) diff --git a/spec/tokenized-buffer-spec.coffee b/spec/tokenized-buffer-spec.coffee index 768e77ba4..0c5363ad3 100644 --- a/spec/tokenized-buffer-spec.coffee +++ b/spec/tokenized-buffer-spec.coffee @@ -580,7 +580,7 @@ describe "TokenizedBuffer", -> describe "when the selector matches a run of multiple tokens at the position", -> it "returns the range covered by all contigous tokens (within a single line)", -> - expect(tokenizedBuffer.bufferRangeForScopeAtPosition('.function', [1, 18])).toEqual [[1, 6], [1, 28]] + expect(tokenizedBuffer.bufferRangeForScopeAtPosition('.meta.function', [1, 18])).toEqual [[1, 6], [1, 28]] describe "when the editor.tabLength config value changes", -> it "updates the tab length of the tokenized lines", -> @@ -734,7 +734,7 @@ describe "TokenizedBuffer", -> it "updates empty line indent guides when the empty line is the last line", -> buffer.insert([12, 2], '\n') - # The newline and he tab need to be in two different operations to surface the bug + # The newline and the tab need to be in two different operations to surface the bug buffer.insert([12, 0], ' ') expect(tokenizedBuffer.tokenizedLineForRow(13).indentLevel).toBe 1 diff --git a/src/token.coffee b/src/token.coffee index 583e25e9b..7b24764ae 100644 --- a/src/token.coffee +++ b/src/token.coffee @@ -24,7 +24,7 @@ class Token {@value, @scopes, @isAtomic, @bufferDelta, @isHardTab, @hasPairedCharacter, @isSoftWrapIndentation} = properties @firstNonWhitespaceIndex = properties.firstNonWhitespaceIndex ? null @firstTrailingWhitespaceIndex = properties.firstTrailingWhitespaceIndex ? null - + @screenDelta = @value.length @bufferDelta ?= @screenDelta @hasPairedCharacter ?= textUtils.hasPairedCharacter(@value) diff --git a/src/tokenized-buffer.coffee b/src/tokenized-buffer.coffee index 9dde39afe..b49f55939 100644 --- a/src/tokenized-buffer.coffee +++ b/src/tokenized-buffer.coffee @@ -1,6 +1,7 @@ _ = require 'underscore-plus' {CompositeDisposable, Emitter} = require 'event-kit' {Point, Range} = require 'text-buffer' +{ScopeSelector} = require 'first-mate' Serializable = require 'serializable' Model = require './model' TokenizedLine = require './tokenized-line' @@ -390,25 +391,53 @@ class TokenizedBuffer extends Model new Point(row, column) bufferRangeForScopeAtPosition: (selector, position) -> + selector = new ScopeSelector(selector.replace(/^\./, '')) position = Point.fromObject(position) - tokenizedLine = @tokenizedLines[position.row] - startIndex = tokenizedLine.tokenIndexAtBufferColumn(position.column) - for index in [startIndex..0] - token = tokenizedLine.tokenAtIndex(index) - break unless token.matchesScopeSelector(selector) - firstToken = token + {parentScopes, tags} = @tokenizedLines[position.row] + scopes = parentScopes.map (tag) -> atom.grammars.scopeForId(tag) - for index in [startIndex...tokenizedLine.getTokenCount()] - token = tokenizedLine.tokenAtIndex(index) - break unless token.matchesScopeSelector(selector) - lastToken = token + startColumn = 0 + for tag, tokenIndex in tags + if tag < 0 + if tag % 2 is -1 + scopes.push(atom.grammars.scopeForId(tag)) + else + scopes.pop() + else + endColumn = startColumn + tag + if endColumn > position.column + break + else + startColumn = endColumn - return unless firstToken? and lastToken? + return unless selector.matches(scopes) - startColumn = tokenizedLine.bufferColumnForToken(firstToken) - endColumn = tokenizedLine.bufferColumnForToken(lastToken) + lastToken.bufferDelta - new Range([position.row, startColumn], [position.row, endColumn]) + startScopes = scopes.slice() + for startTokenIndex in [(tokenIndex - 1)..0] by -1 + tag = tags[startTokenIndex] + if tag < 0 + if tag % 2 is -1 + startScopes.pop() + else + startScopes.push(atom.grammars.scopeForId(tag)) + else + break unless selector.matches(startScopes) + startColumn -= tag + + endScopes = scopes.slice() + for endTokenIndex in [(tokenIndex + 1)...tags.length] by 1 + tag = tags[endTokenIndex] + if tag < 0 + if tag % 2 is -1 + endScopes.push(atom.grammars.scopeForId(tag)) + else + endScopes.pop() + else + break unless selector.matches(endScopes) + endColumn += tag + + new Range(new Point(position.row, startColumn), new Point(position.row, endColumn)) iterateTokensInBufferRange: (bufferRange, iterator) -> bufferRange = Range.fromObject(bufferRange) diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index 063d2f6da..160c8360e 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -459,12 +459,6 @@ class TokenizedLine getTokenCount: -> @tokens.length - bufferColumnForToken: (targetToken) -> - column = 0 - for token in @tokens - return column if token is targetToken - column += token.bufferDelta - getScopeTree: -> return @scopeTree if @scopeTree? From 70eb7f77b018b0b7154e9369364e69f26a498996 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Tue, 12 May 2015 00:12:28 +0200 Subject: [PATCH 11/50] Drop unused methods --- src/tokenized-line.coffee | 43 --------------------------------------- 1 file changed, 43 deletions(-) diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index 160c8360e..be31068b0 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -387,49 +387,6 @@ class TokenizedLine delta = nextDelta delta - breakOutAtomicTokens: (inputTokens) -> - outputTokens = [] - breakOutLeadingSoftTabs = true - column = @startBufferColumn - for token in inputTokens - newTokens = token.breakOutAtomicTokens(@tabLength, breakOutLeadingSoftTabs, column) - column += newToken.value.length for newToken in newTokens - outputTokens.push(newTokens...) - breakOutLeadingSoftTabs = token.isOnlyWhitespace() if breakOutLeadingSoftTabs - outputTokens - - markLeadingAndTrailingWhitespaceTokens: -> - @firstNonWhitespaceIndex = @text.search(NonWhitespaceRegex) - if @firstNonWhitespaceIndex > 0 and isPairedCharacter(@text, @firstNonWhitespaceIndex - 1) - @firstNonWhitespaceIndex-- - @firstTrailingWhitespaceIndex = @text.search(TrailingWhitespaceRegex) - @lineIsWhitespaceOnly = @firstTrailingWhitespaceIndex is 0 - - substituteInvisibleCharacters: -> - invisibles = @invisibles - changedText = false - - for token, i in @tokens - if token.isHardTab - if invisibles.tab - token.value = invisibles.tab + token.value.substring(invisibles.tab.length) - token.hasInvisibleCharacters = true - changedText = true - else - if invisibles.space - if token.hasLeadingWhitespace() and not token.isSoftWrapIndentation - token.value = token.value.replace LeadingWhitespaceRegex, (leadingWhitespace) -> - leadingWhitespace.replace RepeatedSpaceRegex, invisibles.space - token.hasInvisibleCharacters = true - changedText = true - if token.hasTrailingWhitespace() - token.value = token.value.replace TrailingWhitespaceRegex, (leadingWhitespace) -> - leadingWhitespace.replace RepeatedSpaceRegex, invisibles.space - token.hasInvisibleCharacters = true - changedText = true - - @text = @buildText() if changedText - buildEndOfLineInvisibles: -> @endOfLineInvisibles = [] {cr, eol} = @invisibles From b8895cdaaf02b499b7d1345b5a8988b9bc669864 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Tue, 12 May 2015 17:57:51 +0200 Subject: [PATCH 12/50] Update spec based on new interface for Grammar::tokenizeLine --- spec/text-editor-spec.coffee | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spec/text-editor-spec.coffee b/spec/text-editor-spec.coffee index dd6b69f26..66aedcf53 100644 --- a/spec/text-editor-spec.coffee +++ b/spec/text-editor-spec.coffee @@ -4112,8 +4112,9 @@ describe "TextEditor", -> runs -> grammar = atom.grammars.selectGrammar("text.js") - {tokens} = grammar.tokenizeLine("var i; // http://github.com") + {line, tags} = grammar.tokenizeLine("var i; // http://github.com") + tokens = atom.grammars.decodeContent(line, tags) expect(tokens[0].value).toBe "var" expect(tokens[0].scopes).toEqual ["source.js", "storage.modifier.js"] From 77e8a906c230fa791a6c5ec6cbcce287ee90261b Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Tue, 12 May 2015 17:58:16 +0200 Subject: [PATCH 13/50] Fix soft wrapping with tags array, including soft-wrap indent --- src/tokenized-line.coffee | 226 +++++++++++++++++++++++++------------- 1 file changed, 149 insertions(+), 77 deletions(-) diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index be31068b0..352ca1264 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -5,6 +5,7 @@ Token = require './token' SoftTab = Symbol('SoftTab') HardTab = Symbol('HardTab') PairedCharacter = Symbol('PairedCharacter') +SoftWrapIndent = Symbol('SoftWrapIndent') NonWhitespaceRegex = /\S/ LeadingWhitespaceRegex = /^\s*/ @@ -34,9 +35,6 @@ class TokenizedLine @subdivideTokens() @buildEndOfLineInvisibles() if @invisibles? and @lineEnding? - # @softWrapIndentationTokens = @getSoftWrapIndentationTokens() - # @softWrapIndentationDelta = @buildSoftWrapIndentationDelta() - subdivideTokens: -> text = '' bufferColumn = 0 @@ -160,6 +158,9 @@ class TokenizedLine when PairedCharacter tokenProperties.isAtomic = true tokenProperties.hasPairedCharacter = true + when SoftWrapIndent + tokenProperties.isAtomic = true + tokenProperties.isSoftWrapIndentation = true if offset < @firstNonWhitespaceIndex tokenProperties.firstNonWhitespaceIndex = @@ -221,7 +222,7 @@ class TokenizedLine tokenStartColumn += token.screenDelta if @isColumnInsideSoftWrapIndentation(tokenStartColumn) - @softWrapIndentationDelta + @getSoftWrapIndentationDelta() else if token.isAtomic and tokenStartColumn < column if clip is 'forward' tokenStartColumn + token.screenDelta @@ -235,24 +236,59 @@ class TokenizedLine else column - screenColumnForBufferColumn: (bufferColumn, options) -> - bufferColumn = bufferColumn - @startBufferColumn - screenColumn = 0 - currentBufferColumn = 0 - for token in @tokens - break if currentBufferColumn + token.bufferDelta > bufferColumn - screenColumn += token.screenDelta - currentBufferColumn += token.bufferDelta - @clipScreenColumn(screenColumn + (bufferColumn - currentBufferColumn)) - - bufferColumnForScreenColumn: (screenColumn, options) -> + screenColumnForBufferColumn: (targetBufferColumn, options) -> bufferColumn = @startBufferColumn - currentScreenColumn = 0 - for token in @tokens - break if currentScreenColumn + token.screenDelta > screenColumn - bufferColumn += token.bufferDelta - currentScreenColumn += token.screenDelta - bufferColumn + (screenColumn - currentScreenColumn) + screenColumn = 0 + for tag, index in @tags + if tag > 0 + switch @specialTokens[index] + when HardTab + bufferDelta = 1 + screenDelta = tag + when SoftWrapIndent + bufferDelta = 0 + screenDelta = tag + else + bufferDelta = screenDelta = tag + + nextBufferColumn = bufferColumn + bufferDelta + if nextBufferColumn > targetBufferColumn + overshoot = targetBufferColumn - bufferColumn + bufferColumn += overshoot + screenColumn += Math.min(screenDelta, overshoot) + break + else + bufferColumn = nextBufferColumn + screenColumn += screenDelta + + screenColumn + + bufferColumnForScreenColumn: (targetScreenColumn) -> + bufferColumn = @startBufferColumn + screenColumn = 0 + for tag, index in @tags + if tag > 0 + switch @specialTokens[index] + when HardTab + bufferDelta = 1 + screenDelta = tag + when SoftWrapIndent + bufferDelta = 0 + screenDelta = tag + else + bufferDelta = screenDelta = tag + + nextScreenColumn = screenColumn + screenDelta + if nextScreenColumn > targetScreenColumn + overshoot = targetScreenColumn - screenColumn + screenColumn += overshoot + bufferColumn += Math.min(bufferDelta, overshoot) + break + else + screenColumn = nextScreenColumn + bufferColumn += bufferDelta + + bufferColumn getMaxScreenColumn: -> if @fold @@ -286,17 +322,6 @@ class TokenizedLine return maxColumn - buildSoftWrapIndentationTokens: (token, hangingIndent) -> - totalIndentSpaces = (@indentLevel * @tabLength) + hangingIndent - indentTokens = [] - while totalIndentSpaces > 0 - tokenLength = Math.min(@tabLength, totalIndentSpaces) - indentToken = token.buildSoftWrapIndentationToken(tokenLength) - indentTokens.push(indentToken) - totalIndentSpaces -= tokenLength - - indentTokens - softWrapAt: (column, hangingIndent) -> return [null, this] if column is 0 @@ -304,70 +329,117 @@ class TokenizedLine rightText = @text.substring(column) leftTags = [] + rightTags = [] + + leftSpecialTokens = {} + rightSpecialTokens = {} + rightParentScopes = @parentScopes.slice() screenColumn = 0 + for tag, index in @tags + # tag represents a token if tag >= 0 - if screenColumn + tag < column + # token ends before the soft wrap column + if screenColumn + tag <= column + if specialToken = @specialTokens[index] + leftSpecialTokens[index] = specialToken + leftTags.push(tag) screenColumn += tag - else - leftTags.push(column - screenColumn) - rightTags = @tags.slice(index + 1) + + # token starts before and ends after the split column + else if screenColumn <= column + leftSuffix = column - screenColumn rightPrefix = screenColumn + tag - column - rightTags.unshift(rightPrefix) if rightPrefix > 0 - break + + leftTags.push(leftSuffix) if leftSuffix > 0 + + softWrapIndent = @indentLevel * @tabLength + (hangingIndent ? 0) + rightText = ' ' + rightText for i in [0...softWrapIndent] by 1 + while softWrapIndent > 0 + indentToken = Math.min(softWrapIndent, @tabLength) + rightSpecialTokens[rightTags.length] = SoftWrapIndent + rightTags.push(indentToken) + softWrapIndent -= indentToken + + rightTags.push(rightPrefix) if rightPrefix > 0 + + screenColumn += tag + + # token is after split column + else + if specialToken = @specialTokens[index] + rightSpecialTokens[rightTags.length] = specialToken + rightTags.push(tag) + + # tag represents the start or end of a scop else if (tag % 2) is -1 - rightParentScopes.push(tag) + if screenColumn < column + leftTags.push(tag) + rightParentScopes.push(tag) + else + rightTags.push(tag) else - rightParentScopes.pop() - leftTags.push(tag) + if screenColumn < column + leftTags.push(tag) + rightParentScopes.pop() + else + rightTags.push(tag) - softWrapIndent = @indentLevel * @tabLength + (hangingIndent ? 0) - rightTags.unshift(softWrapIndent) if softWrapIndent > 0 + splitBufferColumn = @bufferColumnForScreenColumn(column) + + leftFragment = new TokenizedLine + leftFragment.parentScopes = @parentScopes + leftFragment.text = leftText + leftFragment.tags = leftTags + leftFragment.specialTokens = leftSpecialTokens + leftFragment.startBufferColumn = @startBufferColumn + leftFragment.bufferDelta = splitBufferColumn - @startBufferColumn + leftFragment.ruleStack = @ruleStack + leftFragment.invisibles = @invisibles + leftFragment.lineEnding = null + leftFragment.indentLevel = @indentLevel + leftFragment.tabLength = @tabLength + leftFragment.firstNonWhitespaceIndex = Math.min(column, @firstNonWhitespaceIndex) + leftFragment.firstTrailingWhitespaceIndex = Math.min(column, @firstTrailingWhitespaceIndex) + + rightFragment = new TokenizedLine + rightFragment.parentScopes = rightParentScopes + rightFragment.text = rightText + rightFragment.tags = rightTags + rightFragment.specialTokens = rightSpecialTokens + rightFragment.startBufferColumn = splitBufferColumn + rightFragment.bufferDelta = @bufferDelta - splitBufferColumn + rightFragment.ruleStack = @ruleStack + rightFragment.invisibles = @invisibles + rightFragment.lineEnding = @lineEnding + rightFragment.indentLevel = @indentLevel + rightFragment.tabLength = @tabLength + rightFragment.endOfLineInvisibles = @endOfLineInvisibles + rightFragment.firstNonWhitespaceIndex = Math.max(0, @firstNonWhitespaceIndex - column) + rightFragment.firstTrailingWhitespaceIndex = Math.max(0, @firstTrailingWhitespaceIndex - column) - leftFragment = new TokenizedLine( - parentScopes: @parentScopes - text: leftText - tags: leftTags - startBufferColumn: @startBufferColumn - ruleStack: @ruleStack - invisibles: @invisibles - lineEnding: null, - indentLevel: @indentLevel, - tabLength: @tabLength - ) - rightFragment = new TokenizedLine( - parentScopes: rightParentScopes - text: rightText - tags: rightTags - startBufferColumn: @bufferColumnForScreenColumn(column) - ruleStack: @ruleStack - invisibles: @invisibles - lineEnding: @lineEnding - indentLevel: @indentLevel - tabLength: @tabLength - ) [leftFragment, rightFragment] isSoftWrapped: -> @lineEnding is null - isColumnInsideSoftWrapIndentation: (column) -> - false - # return false if @softWrapIndentationTokens.length is 0 - # - # column < @softWrapIndentationDelta + isColumnInsideSoftWrapIndentation: (targetColumn) -> + targetColumn < @getSoftWrapIndentationDelta() - getSoftWrapIndentationTokens: -> - [] - # _.select(@tokens, (token) -> token.isSoftWrapIndentation) - - buildSoftWrapIndentationDelta: -> - _.reduce @softWrapIndentationTokens, ((acc, token) -> acc + token.screenDelta), 0 + getSoftWrapIndentationDelta: -> + delta = 0 + for tag, index in @tags + if tag >= 0 + if @specialTokens[index] is SoftWrapIndent + delta += tag + else + break + delta hasOnlySoftWrapIndentation: -> - @tokens.length is @softWrapIndentationTokens.length + @getSoftWrapIndentationDelta() is @text.length tokenAtBufferColumn: (bufferColumn) -> @tokens[@tokenIndexAtBufferColumn(bufferColumn)] From 2afe013f9ef514b6144e7c30a5bda827f4d13db3 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Tue, 12 May 2015 18:30:33 +0200 Subject: [PATCH 14/50] Assign ::hasInvisibleCharacters on Token shims --- src/token.coffee | 3 ++- src/tokenized-line.coffee | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/token.coffee b/src/token.coffee index 7b24764ae..6a6683a73 100644 --- a/src/token.coffee +++ b/src/token.coffee @@ -21,7 +21,8 @@ class Token hasInvisibleCharacters: false constructor: (properties) -> - {@value, @scopes, @isAtomic, @bufferDelta, @isHardTab, @hasPairedCharacter, @isSoftWrapIndentation} = properties + {@value, @scopes, @isAtomic, @isHardTab, @bufferDelta} = properties + {@hasInvisibleCharacters, @hasPairedCharacter, @isSoftWrapIndentation} = properties @firstNonWhitespaceIndex = properties.firstNonWhitespaceIndex ? null @firstTrailingWhitespaceIndex = properties.firstTrailingWhitespaceIndex ? null diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index 352ca1264..10ed3a27e 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -155,6 +155,7 @@ class TokenizedLine when HardTab tokenProperties.isAtomic = true tokenProperties.bufferDelta = 1 + tokenProperties.hasInvisibleCharacters = true if @invisibles?.tab when PairedCharacter tokenProperties.isAtomic = true tokenProperties.hasPairedCharacter = true @@ -165,10 +166,12 @@ class TokenizedLine if offset < @firstNonWhitespaceIndex tokenProperties.firstNonWhitespaceIndex = Math.min(tokenProperties.value.length, @firstNonWhitespaceIndex - offset) + tokenProperties.hasInvisibleCharacters = true if @invisibles?.space if @lineEnding? and (offset + tokenProperties.value.length > @firstTrailingWhitespaceIndex) tokenProperties.firstTrailingWhitespaceIndex = Math.max(0, @firstTrailingWhitespaceIndex - offset) + tokenProperties.hasInvisibleCharacters = true if @invisibles?.space offset += tokenProperties.value.length From 75112a017d5f68c86eaec1f61d1087c39d50356e Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Tue, 12 May 2015 18:41:48 +0200 Subject: [PATCH 15/50] Fix assignment of last non-whitespace column for paired characters --- src/tokenized-line.coffee | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index 10ed3a27e..55adcac89 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -67,10 +67,10 @@ class TokenizedLine @tags.splice(tokenIndex, 1, splitTokens...) firstNonWhitespaceColumn ?= screenColumn - lastNonWhitespaceColumn = screenColumn + lastNonWhitespaceColumn = screenColumn + 1 text += @text.substr(bufferColumn, 2) - screenColumn++ + screenColumn += 2 bufferColumn += 2 tokenIndex++ if prefix > 0 From 6befa0200f904fb753a1118142aec40f98081916 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Tue, 12 May 2015 19:36:32 +0200 Subject: [PATCH 16/50] Assign ::isHardTab to shim tokens --- src/tokenized-line.coffee | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index 55adcac89..80e03caac 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -154,6 +154,7 @@ class TokenizedLine tokenProperties.isAtomic = true when HardTab tokenProperties.isAtomic = true + tokenProperties.isHardTab = true tokenProperties.bufferDelta = 1 tokenProperties.hasInvisibleCharacters = true if @invisibles?.tab when PairedCharacter From d90d1f0ea7baa713166ff126ae408b2dc3a2455b Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Tue, 12 May 2015 19:36:53 +0200 Subject: [PATCH 17/50] =?UTF-8?q?Don=E2=80=99t=20build=20::specialTokens?= =?UTF-8?q?=20unless=20properties=20are=20provided?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/tokenized-line.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index 80e03caac..5b13c1f80 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -22,10 +22,10 @@ class TokenizedLine constructor: (properties) -> @id = idCounter++ - @specialTokens = {} return unless properties? + @specialTokens = {} {@parentScopes, @text, @tags, @lineEnding, @ruleStack} = properties {@startBufferColumn, @fold, @tabLength, @indentLevel, @invisibles} = properties From 8ab9a9f9bb6b00d5047efa4b1a84b859ed60b0bd Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Tue, 12 May 2015 19:37:26 +0200 Subject: [PATCH 18/50] Account for softWrapIndent when assigning indices on soft wrap line --- src/tokenized-line.coffee | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index 5b13c1f80..ec13af201 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -361,11 +361,12 @@ class TokenizedLine softWrapIndent = @indentLevel * @tabLength + (hangingIndent ? 0) rightText = ' ' + rightText for i in [0...softWrapIndent] by 1 - while softWrapIndent > 0 - indentToken = Math.min(softWrapIndent, @tabLength) + remainingSoftWrapIndent = softWrapIndent + while remainingSoftWrapIndent > 0 + indentToken = Math.min(remainingSoftWrapIndent, @tabLength) rightSpecialTokens[rightTags.length] = SoftWrapIndent rightTags.push(indentToken) - softWrapIndent -= indentToken + remainingSoftWrapIndent -= indentToken rightTags.push(rightPrefix) if rightPrefix > 0 @@ -421,8 +422,8 @@ class TokenizedLine rightFragment.indentLevel = @indentLevel rightFragment.tabLength = @tabLength rightFragment.endOfLineInvisibles = @endOfLineInvisibles - rightFragment.firstNonWhitespaceIndex = Math.max(0, @firstNonWhitespaceIndex - column) - rightFragment.firstTrailingWhitespaceIndex = Math.max(0, @firstTrailingWhitespaceIndex - column) + rightFragment.firstNonWhitespaceIndex = Math.max(softWrapIndent, @firstNonWhitespaceIndex - column + softWrapIndent) + rightFragment.firstTrailingWhitespaceIndex = Math.max(softWrapIndent, @firstTrailingWhitespaceIndex - column + softWrapIndent) [leftFragment, rightFragment] From 27657537911ef29686e7cf9459012b80ee54505f Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Tue, 12 May 2015 19:39:10 +0200 Subject: [PATCH 19/50] =?UTF-8?q?Don=E2=80=99t=20assume=20same=20token=20i?= =?UTF-8?q?nstances=20in=20spec=20now=20that=20we=20use=20shims?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- spec/tokenized-line-spec.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/tokenized-line-spec.coffee b/spec/tokenized-line-spec.coffee index 0da83c91c..59270179c 100644 --- a/spec/tokenized-line-spec.coffee +++ b/spec/tokenized-line-spec.coffee @@ -27,7 +27,7 @@ describe "TokenizedLine", -> for child in scopeTree.children ensureValidScopeTree(child, scopeDescriptor.concat([scopeTree.scope])) else - expect(scopeTree).toBe tokens[tokenIndex++] + expect(scopeTree).toEqual tokens[tokenIndex++] expect(scopeDescriptor).toEqual scopeTree.scopes waitsForPromise -> From bf6754981b95493d81c8462ef94d59e8ce8cb264 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Tue, 12 May 2015 21:05:33 +0200 Subject: [PATCH 20/50] decodeContent -> decodeTokens --- spec/text-editor-spec.coffee | 2 +- src/tokenized-line.coffee | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/text-editor-spec.coffee b/spec/text-editor-spec.coffee index 66aedcf53..01281492a 100644 --- a/spec/text-editor-spec.coffee +++ b/spec/text-editor-spec.coffee @@ -4114,7 +4114,7 @@ describe "TextEditor", -> grammar = atom.grammars.selectGrammar("text.js") {line, tags} = grammar.tokenizeLine("var i; // http://github.com") - tokens = atom.grammars.decodeContent(line, tags) + tokens = atom.grammars.decodeTokens(line, tags) expect(tokens[0].value).toBe "var" expect(tokens[0].scopes).toEqual ["source.js", "storage.modifier.js"] diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index ec13af201..66d426d7c 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -148,7 +148,7 @@ class TokenizedLine Object.defineProperty @prototype, 'tokens', get: -> offset = 0 - atom.grammars.decodeContent @text, @tags, @parentScopes.slice(), (tokenProperties, index) => + atom.grammars.decodeTokens @text, @tags, @parentScopes.slice(), (tokenProperties, index) => switch @specialTokens[index] when SoftTab tokenProperties.isAtomic = true From 6b51b5d02a708c5bfa6f179492548fcda5aca429 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Wed, 13 May 2015 19:53:22 +0200 Subject: [PATCH 21/50] Rename parentScopes to openScopes --- src/tokenized-buffer.coffee | 36 ++++++++++++++++++------------------ src/tokenized-line.coffee | 16 ++++++++-------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/tokenized-buffer.coffee b/src/tokenized-buffer.coffee index b49f55939..b33fdd8fc 100644 --- a/src/tokenized-buffer.coffee +++ b/src/tokenized-buffer.coffee @@ -154,7 +154,7 @@ class TokenizedBuffer extends Model row = startRow loop previousStack = @stackForRow(row) - @tokenizedLines[row] = @buildTokenizedLineForRow(row, @stackForRow(row - 1), @parentScopesForRow(row)) + @tokenizedLines[row] = @buildTokenizedLineForRow(row, @stackForRow(row - 1), @openScopesForRow(row)) if --rowsRemaining is 0 filledRegion = false endRow = row @@ -214,7 +214,7 @@ class TokenizedBuffer extends Model @updateInvalidRows(start, end, delta) previousEndStack = @stackForRow(end) # used in spill detection below - newTokenizedLines = @buildTokenizedLinesForRows(start, end + delta, @stackForRow(start - 1), @parentScopesForRow(start)) + newTokenizedLines = @buildTokenizedLinesForRows(start, end + delta, @stackForRow(start - 1), @openScopesForRow(start)) _.spliceWithArray(@tokenizedLines, start, end - start + 1, newTokenizedLines) start = @retokenizeWhitespaceRowsIfIndentLevelChanged(start - 1, -1) @@ -235,7 +235,7 @@ class TokenizedBuffer extends Model line = @tokenizedLines[row] if line?.isOnlyWhitespace() and @indentLevelForRow(row) isnt line.indentLevel while line?.isOnlyWhitespace() - @tokenizedLines[row] = @buildTokenizedLineForRow(row, @stackForRow(row - 1), @parentScopesForRow(row)) + @tokenizedLines[row] = @buildTokenizedLineForRow(row, @stackForRow(row - 1), @openScopesForRow(row)) row += increment line = @tokenizedLines[row] @@ -277,17 +277,17 @@ class TokenizedBuffer extends Model @tokenizedLineForRow(row).isComment() and @tokenizedLineForRow(nextRow).isComment() - buildTokenizedLinesForRows: (startRow, endRow, startingStack, startingParentScopes) -> + buildTokenizedLinesForRows: (startRow, endRow, startingStack, startingopenScopes) -> ruleStack = startingStack - parentScopes = startingParentScopes + openScopes = startingopenScopes stopTokenizingAt = startRow + @chunkSize tokenizedLines = for row in [startRow..endRow] if (ruleStack or row is 0) and row < stopTokenizingAt - tokenizedLine = @buildTokenizedLineForRow(row, ruleStack, parentScopes) + tokenizedLine = @buildTokenizedLineForRow(row, ruleStack, openScopes) ruleStack = tokenizedLine.ruleStack - parentScopes = @scopesFromTags(parentScopes, tokenizedLine.tags) + openScopes = @scopesFromTags(openScopes, tokenizedLine.tags) else - tokenizedLine = @buildPlaceholderTokenizedLineForRow(row, parentScopes) + tokenizedLine = @buildPlaceholderTokenizedLineForRow(row, openScopes) tokenizedLine if endRow >= stopTokenizingAt @@ -300,23 +300,23 @@ class TokenizedBuffer extends Model @buildPlaceholderTokenizedLineForRow(row) for row in [startRow..endRow] buildPlaceholderTokenizedLineForRow: (row) -> - parentScopes = [@grammar.idForScope(@grammar.scopeName)] + openScopes = [@grammar.idForScope(@grammar.scopeName)] text = @buffer.lineForRow(row) tags = [text.length] tabLength = @getTabLength() indentLevel = @indentLevelForRow(row) lineEnding = @buffer.lineEndingForRow(row) - new TokenizedLine({parentScopes, text, tags, tabLength, indentLevel, @invisibles, lineEnding}) + new TokenizedLine({openScopes, text, tags, tabLength, indentLevel, @invisibles, lineEnding}) - buildTokenizedLineForRow: (row, ruleStack, parentScopes) -> - @buildTokenizedLineForRowWithText(row, @buffer.lineForRow(row), ruleStack, parentScopes) + buildTokenizedLineForRow: (row, ruleStack, openScopes) -> + @buildTokenizedLineForRowWithText(row, @buffer.lineForRow(row), ruleStack, openScopes) - buildTokenizedLineForRowWithText: (row, text, ruleStack = @stackForRow(row - 1), parentScopes = @parentScopesForRow(row)) -> + buildTokenizedLineForRowWithText: (row, text, ruleStack = @stackForRow(row - 1), openScopes = @openScopesForRow(row)) -> lineEnding = @buffer.lineEndingForRow(row) tabLength = @getTabLength() indentLevel = @indentLevelForRow(row) {tags, ruleStack} = @grammar.tokenizeLine(text, ruleStack, row is 0) - new TokenizedLine({parentScopes, text, tags, ruleStack, tabLength, lineEnding, indentLevel, @invisibles}) + new TokenizedLine({openScopes, text, tags, ruleStack, tabLength, lineEnding, indentLevel, @invisibles}) tokenizedLineForRow: (bufferRow) -> @tokenizedLines[bufferRow] @@ -324,10 +324,10 @@ class TokenizedBuffer extends Model stackForRow: (bufferRow) -> @tokenizedLines[bufferRow]?.ruleStack - parentScopesForRow: (bufferRow) -> + openScopesForRow: (bufferRow) -> if bufferRow > 0 precedingLine = @tokenizedLines[bufferRow - 1] - @scopesFromTags(precedingLine.parentScopes, precedingLine.tags) + @scopesFromTags(precedingLine.openScopes, precedingLine.tags) else [] @@ -394,8 +394,8 @@ class TokenizedBuffer extends Model selector = new ScopeSelector(selector.replace(/^\./, '')) position = Point.fromObject(position) - {parentScopes, tags} = @tokenizedLines[position.row] - scopes = parentScopes.map (tag) -> atom.grammars.scopeForId(tag) + {openScopes, tags} = @tokenizedLines[position.row] + scopes = openScopes.map (tag) -> atom.grammars.scopeForId(tag) startColumn = 0 for tag, tokenIndex in tags diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index 66d426d7c..debf4dea0 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -26,7 +26,7 @@ class TokenizedLine return unless properties? @specialTokens = {} - {@parentScopes, @text, @tags, @lineEnding, @ruleStack} = properties + {@openScopes, @text, @tags, @lineEnding, @ruleStack} = properties {@startBufferColumn, @fold, @tabLength, @indentLevel, @invisibles} = properties @startBufferColumn ?= 0 @@ -148,7 +148,7 @@ class TokenizedLine Object.defineProperty @prototype, 'tokens', get: -> offset = 0 - atom.grammars.decodeTokens @text, @tags, @parentScopes.slice(), (tokenProperties, index) => + atom.grammars.decodeTokens @text, @tags, @openScopes.slice(), (tokenProperties, index) => switch @specialTokens[index] when SoftTab tokenProperties.isAtomic = true @@ -191,7 +191,7 @@ class TokenizedLine copy: -> copy = new TokenizedLine copy.indentLevel = @indentLevel - copy.parentScopes = @parentScopes + copy.openScopes = @openScopes copy.text = @text copy.tags = @tags copy.specialTokens = @specialTokens @@ -338,7 +338,7 @@ class TokenizedLine leftSpecialTokens = {} rightSpecialTokens = {} - rightParentScopes = @parentScopes.slice() + rightopenScopes = @openScopes.slice() screenColumn = 0 @@ -382,20 +382,20 @@ class TokenizedLine else if (tag % 2) is -1 if screenColumn < column leftTags.push(tag) - rightParentScopes.push(tag) + rightopenScopes.push(tag) else rightTags.push(tag) else if screenColumn < column leftTags.push(tag) - rightParentScopes.pop() + rightopenScopes.pop() else rightTags.push(tag) splitBufferColumn = @bufferColumnForScreenColumn(column) leftFragment = new TokenizedLine - leftFragment.parentScopes = @parentScopes + leftFragment.openScopes = @openScopes leftFragment.text = leftText leftFragment.tags = leftTags leftFragment.specialTokens = leftSpecialTokens @@ -410,7 +410,7 @@ class TokenizedLine leftFragment.firstTrailingWhitespaceIndex = Math.min(column, @firstTrailingWhitespaceIndex) rightFragment = new TokenizedLine - rightFragment.parentScopes = rightParentScopes + rightFragment.openScopes = rightopenScopes rightFragment.text = rightText rightFragment.tags = rightTags rightFragment.specialTokens = rightSpecialTokens From d7f558890425ad749503234b012d317ef52bb75c Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Wed, 13 May 2015 21:17:29 +0200 Subject: [PATCH 22/50] Generate line HTML based on tags instead of tokens This avoids creating a bunch of tokens as temporary objects since they are no longer stored. --- src/lines-component.coffee | 146 ++++++++++++++++++++++++------- src/special-token-symbols.coffee | 6 ++ src/text-editor-presenter.coffee | 8 +- src/tokenized-line.coffee | 6 +- 4 files changed, 127 insertions(+), 39 deletions(-) create mode 100644 src/special-token-symbols.coffee diff --git a/src/lines-component.coffee b/src/lines-component.coffee index fbec40b79..9904d5702 100644 --- a/src/lines-component.coffee +++ b/src/lines-component.coffee @@ -4,10 +4,13 @@ _ = require 'underscore-plus' CursorsComponent = require './cursors-component' HighlightsComponent = require './highlights-component' +{HardTab} = require './special-token-symbols' DummyLineNode = $$(-> @div className: 'line', style: 'position: absolute; visibility: hidden;', => @span 'x')[0] AcceptFilter = {acceptNode: -> NodeFilter.FILTER_ACCEPT} WrapperDiv = document.createElement('div') +TokenTextEscapeRegex = /[&"'<>]/g +MaxTokenLength = 20000 cloneObject = (object) -> clone = {} @@ -167,20 +170,122 @@ class LinesComponent @buildEndOfLineHTML(id) or ' ' buildLineInnerHTML: (id) -> - {indentGuidesVisible} = @newState - {tokens, text, isOnlyWhitespace} = @newState.lines[id] + lineState = @newState.lines[id] + {text, openScopes, tags, specialTokens, invisibles} = lineState + {firstNonWhitespaceIndex, firstTrailingWhitespaceIndex} = lineState + lineIsWhitespaceOnly = firstTrailingWhitespaceIndex is 0 + innerHTML = "" - scopeStack = [] - for token in tokens - innerHTML += @updateScopeStack(scopeStack, token.scopes) - hasIndentGuide = indentGuidesVisible and (token.hasLeadingWhitespace() or (token.hasTrailingWhitespace() and isOnlyWhitespace)) - innerHTML += token.getValueAsHtml({hasIndentGuide}) + tokenStart = 0 + scopeDepth = openScopes.length + + for tag in openScopes + scope = atom.grammars.scopeForId(tag) + innerHTML += "" + + for tag, index in tags + # tag represents start or end of a scope + if tag < 0 + if (tag % 2) is -1 + scopeDepth++ + scope = atom.grammars.scopeForId(tag) + innerHTML += "" + else + scopeDepth-- + innerHTML += "" + + # tag represents a token + else + tokenEnd = tokenStart + tag + tokenText = text.substring(tokenStart, tokenEnd) + isHardTab = specialTokens[index] is HardTab + if hasLeadingWhitespace = tokenStart < firstNonWhitespaceIndex + tokenFirstNonWhitespaceIndex = firstNonWhitespaceIndex - tokenStart + else + tokenFirstNonWhitespaceIndex = null + if hasTrailingWhitespace = tokenEnd > firstTrailingWhitespaceIndex + tokenFirstTrailingWhitespaceIndex = Math.max(0, firstTrailingWhitespaceIndex - tokenStart) + else + tokenFirstTrailingWhitespaceIndex = null + hasIndentGuide = @newState.indentGuidesVisible and (hasLeadingWhitespace or lineIsWhitespaceOnly) + hasInvisibleCharacters = + (invisibles?.tab and isHardTab) or + (invisibles?.space and (hasLeadingWhitespace or hasTrailingWhitespace)) + + innerHTML += @buildTokenHTML(tokenText, isHardTab, tokenFirstNonWhitespaceIndex, tokenFirstTrailingWhitespaceIndex, hasIndentGuide, hasInvisibleCharacters) + + tokenStart = tokenEnd + + while scopeDepth > 0 + innerHTML += "" + scopeDepth-- - innerHTML += @popScope(scopeStack) while scopeStack.length > 0 innerHTML += @buildEndOfLineHTML(id) innerHTML + buildTokenHTML: (tokenText, isHardTab, firstNonWhitespaceIndex, firstTrailingWhitespaceIndex, hasIndentGuide, hasInvisibleCharacters) -> + if isHardTab + classes = 'hard-tab' + classes += ' leading-whitespace' if firstNonWhitespaceIndex? + classes += ' trailing-whitespace' if firstTrailingWhitespaceIndex? + classes += ' indent-guide' if hasIndentGuide + classes += ' invisible-character' if hasInvisibleCharacters + return "#{@escapeTokenText(tokenText)}" + else + startIndex = 0 + endIndex = tokenText.length + + leadingHtml = '' + trailingHtml = '' + + if firstNonWhitespaceIndex? + leadingWhitespace = tokenText.substring(0, firstNonWhitespaceIndex) + + classes = 'leading-whitespace' + classes += ' indent-guide' if hasIndentGuide + classes += ' invisible-character' if hasInvisibleCharacters + + leadingHtml = "#{leadingWhitespace}" + startIndex = firstNonWhitespaceIndex + + if firstTrailingWhitespaceIndex? + tokenIsOnlyWhitespace = firstTrailingWhitespaceIndex is 0 + trailingWhitespace = tokenText.substring(firstTrailingWhitespaceIndex) + + classes = 'trailing-whitespace' + classes += ' indent-guide' if hasIndentGuide and not firstNonWhitespaceIndex? and tokenIsOnlyWhitespace + classes += ' invisible-character' if hasInvisibleCharacters + + trailingHtml = "#{trailingWhitespace}" + + endIndex = firstTrailingWhitespaceIndex + + html = leadingHtml + if tokenText.length > MaxTokenLength + while startIndex < endIndex + html += "" + @escapeTokenText(tokenText, startIndex, startIndex + MaxTokenLength) + "" + startIndex += MaxTokenLength + else + html += @escapeTokenText(tokenText, startIndex, endIndex) + + html += trailingHtml + html + + escapeTokenText: (tokenText, startIndex, endIndex) -> + if startIndex? and endIndex? and startIndex > 0 or endIndex < tokenText.length + tokenText = tokenText.slice(startIndex, endIndex) + tokenText.replace(TokenTextEscapeRegex, @escapeTokenTextReplace) + + escapeTokenTextReplace: (match) -> + switch match + when '&' then '&' + when '"' then '"' + when "'" then ''' + when '<' then '<' + when '>' then '>' + else match + buildEndOfLineHTML: (id) -> {endOfLineInvisibles} = @newState.lines[id] @@ -190,31 +295,6 @@ class LinesComponent html += "#{invisible}" html - updateScopeStack: (scopeStack, desiredScopeDescriptor) -> - html = "" - - # Find a common prefix - for scope, i in desiredScopeDescriptor - break unless scopeStack[i] is desiredScopeDescriptor[i] - - # Pop scopeDescriptor until we're at the common prefx - until scopeStack.length is i - html += @popScope(scopeStack) - - # Push onto common prefix until scopeStack equals desiredScopeDescriptor - for j in [i...desiredScopeDescriptor.length] - html += @pushScope(scopeStack, desiredScopeDescriptor[j]) - - html - - popScope: (scopeStack) -> - scopeStack.pop() - "" - - pushScope: (scopeStack, scope) -> - scopeStack.push(scope) - "" - updateLineNode: (id) -> oldLineState = @oldState.lines[id] newLineState = @newState.lines[id] diff --git a/src/special-token-symbols.coffee b/src/special-token-symbols.coffee new file mode 100644 index 000000000..06884b85f --- /dev/null +++ b/src/special-token-symbols.coffee @@ -0,0 +1,6 @@ +module.exports = { + SoftTab: Symbol('SoftTab') + HardTab: Symbol('HardTab') + PairedCharacter: Symbol('PairedCharacter') + SoftWrapIndent: Symbol('SoftWrapIndent') +} diff --git a/src/text-editor-presenter.coffee b/src/text-editor-presenter.coffee index ac6d0c363..d5e8fc09c 100644 --- a/src/text-editor-presenter.coffee +++ b/src/text-editor-presenter.coffee @@ -334,9 +334,15 @@ class TextEditorPresenter @state.content.lines[line.id] = screenRow: row text: line.text + openScopes: line.openScopes + tags: line.tags + specialTokens: line.specialTokens + firstNonWhitespaceIndex: line.firstNonWhitespaceIndex + firstTrailingWhitespaceIndex: line.firstTrailingWhitespaceIndex + invisibles: line.invisibles + endOfLineInvisibles: line.endOfLineInvisibles tokens: line.tokens isOnlyWhitespace: line.isOnlyWhitespace() - endOfLineInvisibles: line.endOfLineInvisibles indentLevel: line.indentLevel tabLength: line.tabLength fold: line.fold diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index debf4dea0..c10c6693d 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -1,11 +1,7 @@ _ = require 'underscore-plus' {isPairedCharacter} = require './text-utils' Token = require './token' - -SoftTab = Symbol('SoftTab') -HardTab = Symbol('HardTab') -PairedCharacter = Symbol('PairedCharacter') -SoftWrapIndent = Symbol('SoftWrapIndent') +{SoftTab, HardTab, PairedCharacter, SoftWrapIndent} = require './special-token-symbols' NonWhitespaceRegex = /\S/ LeadingWhitespaceRegex = /^\s*/ From 00b30f7db8a248c15aa2b5128937c5231ed3ff55 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Wed, 13 May 2015 22:03:26 +0200 Subject: [PATCH 23/50] Remove some dead code --- spec/tokenized-line-spec.coffee | 21 ---- src/token.coffee | 192 -------------------------------- src/tokenized-line.coffee | 32 ------ 3 files changed, 245 deletions(-) diff --git a/spec/tokenized-line-spec.coffee b/spec/tokenized-line-spec.coffee index 59270179c..2914ec089 100644 --- a/spec/tokenized-line-spec.coffee +++ b/spec/tokenized-line-spec.coffee @@ -17,24 +17,3 @@ describe "TokenizedLine", -> it "returns false when the line is not only whitespace", -> expect(editor.tokenizedLineForScreenRow(0).isOnlyWhitespace()).toBe false expect(editor.tokenizedLineForScreenRow(2).isOnlyWhitespace()).toBe false - - describe "::getScopeTree()", -> - it "returns a tree whose inner nodes are scopeDescriptor and whose leaf nodes are tokens in those scopeDescriptor", -> - [tokens, tokenIndex] = [] - - ensureValidScopeTree = (scopeTree, scopeDescriptor=[]) -> - if scopeTree.children? - for child in scopeTree.children - ensureValidScopeTree(child, scopeDescriptor.concat([scopeTree.scope])) - else - expect(scopeTree).toEqual tokens[tokenIndex++] - expect(scopeDescriptor).toEqual scopeTree.scopes - - waitsForPromise -> - atom.project.open('coffee.coffee').then (o) -> editor = o - - runs -> - tokenIndex = 0 - tokens = editor.tokenizedLineForScreenRow(1).tokens - scopeTree = editor.tokenizedLineForScreenRow(1).getScopeTree() - ensureValidScopeTree(scopeTree) diff --git a/src/token.coffee b/src/token.coffee index 6a6683a73..60e8194f8 100644 --- a/src/token.coffee +++ b/src/token.coffee @@ -1,13 +1,8 @@ _ = require 'underscore-plus' -textUtils = require './text-utils' -WhitespaceRegexesByTabLength = {} -EscapeRegex = /[&"'<>]/g StartDotRegex = /^\.?/ WhitespaceRegex = /\S/ -MaxTokenLength = 20000 - # Represents a single unit of text as selected by a grammar. module.exports = class Token @@ -28,7 +23,6 @@ class Token @screenDelta = @value.length @bufferDelta ?= @screenDelta - @hasPairedCharacter ?= textUtils.hasPairedCharacter(@value) isEqual: (other) -> # TODO: scopes is deprecated. This is here for the sake of lang package tests @@ -37,126 +31,6 @@ class Token isBracket: -> /^meta\.brace\b/.test(_.last(@scopes)) - splitAt: (splitIndex) -> - leftToken = new Token(value: @value.substring(0, splitIndex), scopes: @scopes) - rightToken = new Token(value: @value.substring(splitIndex), scopes: @scopes) - - if @firstNonWhitespaceIndex? - leftToken.firstNonWhitespaceIndex = Math.min(splitIndex, @firstNonWhitespaceIndex) - leftToken.hasInvisibleCharacters = @hasInvisibleCharacters - - if @firstNonWhitespaceIndex > splitIndex - rightToken.firstNonWhitespaceIndex = @firstNonWhitespaceIndex - splitIndex - rightToken.hasInvisibleCharacters = @hasInvisibleCharacters - - if @firstTrailingWhitespaceIndex? - rightToken.firstTrailingWhitespaceIndex = Math.max(0, @firstTrailingWhitespaceIndex - splitIndex) - rightToken.hasInvisibleCharacters = @hasInvisibleCharacters - - if @firstTrailingWhitespaceIndex < splitIndex - leftToken.firstTrailingWhitespaceIndex = @firstTrailingWhitespaceIndex - leftToken.hasInvisibleCharacters = @hasInvisibleCharacters - - [leftToken, rightToken] - - whitespaceRegexForTabLength: (tabLength) -> - WhitespaceRegexesByTabLength[tabLength] ?= new RegExp("([ ]{#{tabLength}})|(\t)|([^\t]+)", "g") - - breakOutAtomicTokens: (tabLength, breakOutLeadingSoftTabs, startColumn) -> - if @hasPairedCharacter - outputTokens = [] - column = startColumn - - for token in @breakOutPairedCharacters() - if token.isAtomic - outputTokens.push(token) - else - outputTokens.push(token.breakOutAtomicTokens(tabLength, breakOutLeadingSoftTabs, column)...) - breakOutLeadingSoftTabs = token.isOnlyWhitespace() if breakOutLeadingSoftTabs - column += token.value.length - - outputTokens - else - return [this] if @isAtomic - - if breakOutLeadingSoftTabs - return [this] unless /^[ ]|\t/.test(@value) - else - return [this] unless /\t/.test(@value) - - outputTokens = [] - regex = @whitespaceRegexForTabLength(tabLength) - column = startColumn - while match = regex.exec(@value) - [fullMatch, softTab, hardTab] = match - token = null - if softTab and breakOutLeadingSoftTabs - token = @buildSoftTabToken(tabLength) - else if hardTab - breakOutLeadingSoftTabs = false - token = @buildHardTabToken(tabLength, column) - else - breakOutLeadingSoftTabs = false - value = match[0] - token = new Token({value, @scopes}) - column += token.value.length - outputTokens.push(token) - - outputTokens - - breakOutPairedCharacters: -> - outputTokens = [] - index = 0 - nonPairStart = 0 - - while index < @value.length - if textUtils.isPairedCharacter(@value, index) - if nonPairStart isnt index - outputTokens.push(new Token({value: @value[nonPairStart...index], @scopes})) - outputTokens.push(@buildPairedCharacterToken(@value, index)) - index += 2 - nonPairStart = index - else - index++ - - if nonPairStart isnt index - outputTokens.push(new Token({value: @value[nonPairStart...index], @scopes})) - - outputTokens - - buildPairedCharacterToken: (value, index) -> - new Token( - value: value[index..index + 1] - scopes: @scopes - isAtomic: true - hasPairedCharacter: true - ) - - buildHardTabToken: (tabLength, column) -> - @buildTabToken(tabLength, true, column) - - buildSoftTabToken: (tabLength) -> - @buildTabToken(tabLength, false, 0) - - buildTabToken: (tabLength, isHardTab, column=0) -> - tabStop = tabLength - (column % tabLength) - new Token( - value: _.multiplyString(" ", tabStop) - scopes: @scopes - bufferDelta: if isHardTab then 1 else tabStop - isAtomic: true - isHardTab: isHardTab - ) - - buildSoftWrapIndentationToken: (length) -> - new Token( - value: _.multiplyString(" ", length), - scopes: @scopes, - bufferDelta: 0, - isAtomic: true, - isSoftWrapIndentation: true - ) - isOnlyWhitespace: -> not WhitespaceRegex.test(@value) @@ -166,72 +40,6 @@ class Token scopeClasses = scope.split('.') _.isSubset(targetClasses, scopeClasses) - getValueAsHtml: ({hasIndentGuide}) -> - if @isHardTab - classes = 'hard-tab' - classes += ' leading-whitespace' if @hasLeadingWhitespace() - classes += ' trailing-whitespace' if @hasTrailingWhitespace() - classes += ' indent-guide' if hasIndentGuide - classes += ' invisible-character' if @hasInvisibleCharacters - html = "#{@escapeString(@value)}" - else - startIndex = 0 - endIndex = @value.length - - leadingHtml = '' - trailingHtml = '' - - if @hasLeadingWhitespace() - leadingWhitespace = @value.substring(0, @firstNonWhitespaceIndex) - - classes = 'leading-whitespace' - classes += ' indent-guide' if hasIndentGuide - classes += ' invisible-character' if @hasInvisibleCharacters - - leadingHtml = "#{leadingWhitespace}" - startIndex = @firstNonWhitespaceIndex - - if @hasTrailingWhitespace() - tokenIsOnlyWhitespace = @firstTrailingWhitespaceIndex is 0 - trailingWhitespace = @value.substring(@firstTrailingWhitespaceIndex) - - classes = 'trailing-whitespace' - classes += ' indent-guide' if hasIndentGuide and not @hasLeadingWhitespace() and tokenIsOnlyWhitespace - classes += ' invisible-character' if @hasInvisibleCharacters - - trailingHtml = "#{trailingWhitespace}" - - endIndex = @firstTrailingWhitespaceIndex - - html = leadingHtml - if @value.length > MaxTokenLength - while startIndex < endIndex - html += "" + @escapeString(@value, startIndex, startIndex + MaxTokenLength) + "" - startIndex += MaxTokenLength - else - html += @escapeString(@value, startIndex, endIndex) - - html += trailingHtml - html - - escapeString: (str, startIndex, endIndex) -> - strLength = str.length - - startIndex ?= 0 - endIndex ?= strLength - - str = str.slice(startIndex, endIndex) if startIndex > 0 or endIndex < strLength - str.replace(EscapeRegex, @escapeStringReplace) - - escapeStringReplace: (match) -> - switch match - when '&' then '&' - when '"' then '"' - when "'" then ''' - when '<' then '<' - when '>' then '>' - else match - hasLeadingWhitespace: -> @firstNonWhitespaceIndex? and @firstNonWhitespaceIndex > 0 diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index c10c6693d..99db15c3b 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -488,35 +488,3 @@ class TokenizedLine getTokenCount: -> @tokens.length - - getScopeTree: -> - return @scopeTree if @scopeTree? - - scopeStack = [] - for token in @tokens - @updateScopeStack(scopeStack, token.scopes) - _.last(scopeStack).children.push(token) - - @scopeTree = scopeStack[0] - @updateScopeStack(scopeStack, []) - @scopeTree - - updateScopeStack: (scopeStack, desiredScopeDescriptor) -> - # Find a common prefix - for scope, i in desiredScopeDescriptor - break unless scopeStack[i]?.scope is desiredScopeDescriptor[i] - - # Pop scopeDescriptor until we're at the common prefx - until scopeStack.length is i - poppedScope = scopeStack.pop() - _.last(scopeStack)?.children.push(poppedScope) - - # Push onto common prefix until scopeStack equals desiredScopeDescriptor - for j in [i...desiredScopeDescriptor.length] - scopeStack.push(new Scope(desiredScopeDescriptor[j])) - - return - -class Scope - constructor: (@scope) -> - @children = [] From dc473698a9ef2deedb69664e6a486c5b67030ea6 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Wed, 13 May 2015 22:03:29 +0200 Subject: [PATCH 24/50] :art: --- src/tokenized-line.coffee | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index 99db15c3b..012d68996 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -334,7 +334,7 @@ class TokenizedLine leftSpecialTokens = {} rightSpecialTokens = {} - rightopenScopes = @openScopes.slice() + rightOpenScopes = @openScopes.slice() screenColumn = 0 @@ -378,13 +378,13 @@ class TokenizedLine else if (tag % 2) is -1 if screenColumn < column leftTags.push(tag) - rightopenScopes.push(tag) + rightOpenScopes.push(tag) else rightTags.push(tag) else if screenColumn < column leftTags.push(tag) - rightopenScopes.pop() + rightOpenScopes.pop() else rightTags.push(tag) @@ -406,7 +406,7 @@ class TokenizedLine leftFragment.firstTrailingWhitespaceIndex = Math.min(column, @firstTrailingWhitespaceIndex) rightFragment = new TokenizedLine - rightFragment.openScopes = rightopenScopes + rightFragment.openScopes = rightOpenScopes rightFragment.text = rightText rightFragment.tags = rightTags rightFragment.specialTokens = rightSpecialTokens From a7550666ddf84e8f44c87e2ea948f2a4e9de5e2a Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Wed, 13 May 2015 22:50:05 +0200 Subject: [PATCH 25/50] Remove dead code --- src/tokenized-line.coffee | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index 012d68996..cbc56da99 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -174,16 +174,6 @@ class TokenizedLine new Token(tokenProperties) - buildText: -> - text = "" - text += token.value for token in @tokens - text - - buildBufferDelta: -> - delta = 0 - delta += token.bufferDelta for token in @tokens - delta - copy: -> copy = new TokenizedLine copy.indentLevel = @indentLevel From da2df2297a3944070318f139b1c5a3e4ac2d3fc2 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Thu, 14 May 2015 00:11:26 +0200 Subject: [PATCH 26/50] Add a TokenIterator and use it for tokens shim --- src/token-iterator.coffee | 85 +++++++++++++++++++++++++++++++++++++++ src/tokenized-line.coffee | 56 +++++++++++++------------- 2 files changed, 113 insertions(+), 28 deletions(-) create mode 100644 src/token-iterator.coffee diff --git a/src/token-iterator.coffee b/src/token-iterator.coffee new file mode 100644 index 000000000..a8b9db59e --- /dev/null +++ b/src/token-iterator.coffee @@ -0,0 +1,85 @@ +{SoftTab, HardTab, PairedCharacter, SoftWrapIndent} = require './special-token-symbols' + +module.exports = +class TokenIterator + @instance: new this + + constructor: (line) -> + @reset(line) if line? + + reset: (@line) -> + @index = null + @bufferStart = 0 + @bufferEnd = 0 + @screenStart = 0 + @screenEnd = 0 + @scopes = @line.openScopes.map (id) -> atom.grammars.scopeForId(id) + @scopeStarts = @scopes.slice() + @scopeEnds = [] + this + + next: -> + {tags} = @line + + if @index? + @index++ + @scopeEnds.length = 0 + @scopeStarts.length = 0 + @bufferStart = @bufferEnd + @screenStart = @screenEnd + else + @index = 0 + + while @index < tags.length + tag = tags[@index] + if tag < 0 + if tag % 2 is 0 + @scopeEnds.push(atom.grammars.scopeForId(tag + 1)) + @scopes.pop() + else + scope = atom.grammars.scopeForId(tag) + @scopeStarts.push(scope) + @scopes.push(scope) + @index++ + else + if @isHardTab() + @screenEnd = @screenStart + tag + @bufferEnd = @bufferStart + 1 + else if @isSoftWrapIndentation() + @screenEnd = @screenStart + tag + @bufferEnd = @bufferStart + 0 + else + @screenEnd = @screenStart + tag + @bufferEnd = @bufferStart + tag + return true + + false + + getBufferStart: -> @bufferStart + getBufferEnd: -> @bufferEnd + + getScreenStart: -> @screenStart + getScreenEnd: -> @screenEnd + + getScopeStarts: -> @scopeStarts + getScopeEnds: -> @scopeEnds + + getScopes: -> @scopes + + getText: -> + @line.text.substring(@screenStart, @screenEnd) + + isSoftTab: -> + @line.specialTokens[@index] is SoftTab + + isHardTab: -> + @line.specialTokens[@index] is HardTab + + isSoftWrapIndentation: -> + @line.specialTokens[@index] is SoftWrapIndent + + isPairedCharacter: -> + @line.specialTokens[@index] is PairedCharacter + + isAtomic: -> + @isSoftTab() or @isHardTab() or @isSoftWrapIndentation() or @isPairedCharacter() diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index cbc56da99..de1c45e96 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -1,6 +1,7 @@ _ = require 'underscore-plus' {isPairedCharacter} = require './text-utils' Token = require './token' +TokenIterator = require './token-iterator' {SoftTab, HardTab, PairedCharacter, SoftWrapIndent} = require './special-token-symbols' NonWhitespaceRegex = /\S/ @@ -28,10 +29,10 @@ class TokenizedLine @startBufferColumn ?= 0 @bufferDelta = @text.length - @subdivideTokens() + @transformContent() @buildEndOfLineInvisibles() if @invisibles? and @lineEnding? - subdivideTokens: -> + transformContent: -> text = '' bufferColumn = 0 screenColumn = 0 @@ -142,37 +143,36 @@ class TokenizedLine @firstTrailingWhitespaceIndex = 0 Object.defineProperty @prototype, 'tokens', get: -> - offset = 0 + iterator = TokenIterator.instance.reset(this) + tokens = [] - atom.grammars.decodeTokens @text, @tags, @openScopes.slice(), (tokenProperties, index) => - switch @specialTokens[index] - when SoftTab - tokenProperties.isAtomic = true - when HardTab - tokenProperties.isAtomic = true - tokenProperties.isHardTab = true - tokenProperties.bufferDelta = 1 - tokenProperties.hasInvisibleCharacters = true if @invisibles?.tab - when PairedCharacter - tokenProperties.isAtomic = true - tokenProperties.hasPairedCharacter = true - when SoftWrapIndent - tokenProperties.isAtomic = true - tokenProperties.isSoftWrapIndentation = true + while iterator.next() + properties = { + value: iterator.getText() + scopes: iterator.getScopes().slice() + isAtomic: iterator.isAtomic() + isHardTab: iterator.isHardTab() + hasPairedCharacter: iterator.isPairedCharacter() + isSoftWrapIndentation: iterator.isSoftWrapIndentation() + } - if offset < @firstNonWhitespaceIndex - tokenProperties.firstNonWhitespaceIndex = - Math.min(tokenProperties.value.length, @firstNonWhitespaceIndex - offset) - tokenProperties.hasInvisibleCharacters = true if @invisibles?.space + if iterator.isHardTab() + properties.bufferDelta = 1 + properties.hasInvisibleCharacters = true if @invisibles?.tab - if @lineEnding? and (offset + tokenProperties.value.length > @firstTrailingWhitespaceIndex) - tokenProperties.firstTrailingWhitespaceIndex = - Math.max(0, @firstTrailingWhitespaceIndex - offset) - tokenProperties.hasInvisibleCharacters = true if @invisibles?.space + if iterator.getScreenStart() < @firstNonWhitespaceIndex + properties.firstNonWhitespaceIndex = + Math.min(@firstNonWhitespaceIndex, iterator.getScreenEnd()) - iterator.getScreenStart() + properties.hasInvisibleCharacters = true if @invisibles?.space - offset += tokenProperties.value.length + if @lineEnding? and iterator.getScreenEnd() > @firstTrailingWhitespaceIndex + properties.firstTrailingWhitespaceIndex = + Math.max(0, @firstTrailingWhitespaceIndex - iterator.getScreenStart()) + properties.hasInvisibleCharacters = true if @invisibles?.space - new Token(tokenProperties) + tokens.push(new Token(properties)) + + tokens copy: -> copy = new TokenizedLine From 0eb97e6a962d82eeb156637b8c70c87e2b287d8a Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Thu, 14 May 2015 00:33:27 +0200 Subject: [PATCH 27/50] Use TokenIterator for position translation --- src/token-iterator.coffee | 4 +-- src/tokenized-line.coffee | 72 ++++++++++++--------------------------- 2 files changed, 24 insertions(+), 52 deletions(-) diff --git a/src/token-iterator.coffee b/src/token-iterator.coffee index a8b9db59e..800f76d03 100644 --- a/src/token-iterator.coffee +++ b/src/token-iterator.coffee @@ -9,8 +9,8 @@ class TokenIterator reset: (@line) -> @index = null - @bufferStart = 0 - @bufferEnd = 0 + @bufferStart = @line.startBufferColumn + @bufferEnd = @bufferStart @screenStart = 0 @screenEnd = 0 @scopes = @line.openScopes.map (id) -> atom.grammars.scopeForId(id) diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index de1c45e96..2183ac349 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -227,58 +227,30 @@ class TokenizedLine column screenColumnForBufferColumn: (targetBufferColumn, options) -> - bufferColumn = @startBufferColumn - screenColumn = 0 - for tag, index in @tags - if tag > 0 - switch @specialTokens[index] - when HardTab - bufferDelta = 1 - screenDelta = tag - when SoftWrapIndent - bufferDelta = 0 - screenDelta = tag - else - bufferDelta = screenDelta = tag - - nextBufferColumn = bufferColumn + bufferDelta - if nextBufferColumn > targetBufferColumn - overshoot = targetBufferColumn - bufferColumn - bufferColumn += overshoot - screenColumn += Math.min(screenDelta, overshoot) - break - else - bufferColumn = nextBufferColumn - screenColumn += screenDelta - - screenColumn + iterator = TokenIterator.instance.reset(this) + while iterator.next() + tokenBufferStart = iterator.getBufferStart() + tokenBufferEnd = iterator.getBufferEnd() + if tokenBufferStart <= targetBufferColumn < tokenBufferEnd + overshoot = targetBufferColumn - tokenBufferStart + return Math.min( + iterator.getScreenStart() + overshoot, + iterator.getScreenEnd() + ) + iterator.getScreenEnd() bufferColumnForScreenColumn: (targetScreenColumn) -> - bufferColumn = @startBufferColumn - screenColumn = 0 - for tag, index in @tags - if tag > 0 - switch @specialTokens[index] - when HardTab - bufferDelta = 1 - screenDelta = tag - when SoftWrapIndent - bufferDelta = 0 - screenDelta = tag - else - bufferDelta = screenDelta = tag - - nextScreenColumn = screenColumn + screenDelta - if nextScreenColumn > targetScreenColumn - overshoot = targetScreenColumn - screenColumn - screenColumn += overshoot - bufferColumn += Math.min(bufferDelta, overshoot) - break - else - screenColumn = nextScreenColumn - bufferColumn += bufferDelta - - bufferColumn + iterator = TokenIterator.instance.reset(this) + while iterator.next() + tokenScreenStart = iterator.getScreenStart() + tokenScreenEnd = iterator.getScreenEnd() + if tokenScreenStart <= targetScreenColumn < tokenScreenEnd + overshoot = targetScreenColumn - tokenScreenStart + return Math.min( + iterator.getBufferStart() + overshoot, + iterator.getBufferEnd() + ) + iterator.getBufferEnd() getMaxScreenColumn: -> if @fold From 121e42debac36088b5a1055d26172523377af700 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Thu, 14 May 2015 00:55:06 +0200 Subject: [PATCH 28/50] Use TokenIterator to build line HTML --- src/lines-component.coffee | 71 ++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 38 deletions(-) diff --git a/src/lines-component.coffee b/src/lines-component.coffee index 9904d5702..80b32b88b 100644 --- a/src/lines-component.coffee +++ b/src/lines-component.coffee @@ -4,7 +4,7 @@ _ = require 'underscore-plus' CursorsComponent = require './cursors-component' HighlightsComponent = require './highlights-component' -{HardTab} = require './special-token-symbols' +TokenIterator = require './token-iterator' DummyLineNode = $$(-> @div className: 'line', style: 'position: absolute; visibility: hidden;', => @span 'x')[0] AcceptFilter = {acceptNode: -> NodeFilter.FILTER_ACCEPT} @@ -171,55 +171,50 @@ class LinesComponent buildLineInnerHTML: (id) -> lineState = @newState.lines[id] - {text, openScopes, tags, specialTokens, invisibles} = lineState - {firstNonWhitespaceIndex, firstTrailingWhitespaceIndex} = lineState + {firstNonWhitespaceIndex, firstTrailingWhitespaceIndex, invisibles} = lineState lineIsWhitespaceOnly = firstTrailingWhitespaceIndex is 0 innerHTML = "" + iterator = TokenIterator.instance.reset(lineState) - tokenStart = 0 - scopeDepth = openScopes.length + debugger if global.debug + while iterator.next() + for scope in iterator.getScopeEnds() + innerHTML += "" - for tag in openScopes - scope = atom.grammars.scopeForId(tag) - innerHTML += "" + for scope in iterator.getScopeStarts() + innerHTML += "" - for tag, index in tags - # tag represents start or end of a scope - if tag < 0 - if (tag % 2) is -1 - scopeDepth++ - scope = atom.grammars.scopeForId(tag) - innerHTML += "" - else - scopeDepth-- - innerHTML += "" + tokenStart = iterator.getScreenStart() + tokenEnd = iterator.getScreenEnd() + tokenText = iterator.getText() + isHardTab = iterator.isHardTab() - # tag represents a token + if hasLeadingWhitespace = tokenStart < firstNonWhitespaceIndex + tokenFirstNonWhitespaceIndex = firstNonWhitespaceIndex - tokenStart else - tokenEnd = tokenStart + tag - tokenText = text.substring(tokenStart, tokenEnd) - isHardTab = specialTokens[index] is HardTab - if hasLeadingWhitespace = tokenStart < firstNonWhitespaceIndex - tokenFirstNonWhitespaceIndex = firstNonWhitespaceIndex - tokenStart - else - tokenFirstNonWhitespaceIndex = null - if hasTrailingWhitespace = tokenEnd > firstTrailingWhitespaceIndex - tokenFirstTrailingWhitespaceIndex = Math.max(0, firstTrailingWhitespaceIndex - tokenStart) - else - tokenFirstTrailingWhitespaceIndex = null - hasIndentGuide = @newState.indentGuidesVisible and (hasLeadingWhitespace or lineIsWhitespaceOnly) - hasInvisibleCharacters = - (invisibles?.tab and isHardTab) or - (invisibles?.space and (hasLeadingWhitespace or hasTrailingWhitespace)) + tokenFirstNonWhitespaceIndex = null - innerHTML += @buildTokenHTML(tokenText, isHardTab, tokenFirstNonWhitespaceIndex, tokenFirstTrailingWhitespaceIndex, hasIndentGuide, hasInvisibleCharacters) + if hasTrailingWhitespace = tokenEnd > firstTrailingWhitespaceIndex + tokenFirstTrailingWhitespaceIndex = Math.max(0, firstTrailingWhitespaceIndex - tokenStart) + else + tokenFirstTrailingWhitespaceIndex = null - tokenStart = tokenEnd + hasIndentGuide = + @newState.indentGuidesVisible and + (hasLeadingWhitespace or lineIsWhitespaceOnly) - while scopeDepth > 0 + hasInvisibleCharacters = + (invisibles?.tab and isHardTab) or + (invisibles?.space and (hasLeadingWhitespace or hasTrailingWhitespace)) + + innerHTML += @buildTokenHTML(tokenText, isHardTab, tokenFirstNonWhitespaceIndex, tokenFirstTrailingWhitespaceIndex, hasIndentGuide, hasInvisibleCharacters) + + for scope in iterator.getScopeEnds() + innerHTML += "" + + for scope in iterator.getScopes() innerHTML += "" - scopeDepth-- innerHTML += @buildEndOfLineHTML(id) innerHTML From c76b45206bd781f8ea0350af5e00a5000b872c5a Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Thu, 14 May 2015 01:03:34 +0200 Subject: [PATCH 29/50] Use TokenIterator to compute horizontal pixel positions Instead of the TokenizedLine::tokens shim --- src/text-editor-presenter.coffee | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/text-editor-presenter.coffee b/src/text-editor-presenter.coffee index d5e8fc09c..9be8e9bc0 100644 --- a/src/text-editor-presenter.coffee +++ b/src/text-editor-presenter.coffee @@ -2,6 +2,7 @@ {Point, Range} = require 'text-buffer' _ = require 'underscore-plus' Decoration = require './decoration' +TokenIterator = require './token-iterator' module.exports = class TextEditorPresenter @@ -980,17 +981,20 @@ class TextEditorPresenter top = targetRow * @lineHeight left = 0 column = 0 - for token in @model.tokenizedLineForScreenRow(targetRow).tokens - characterWidths = @getScopedCharacterWidths(token.scopes) + + iterator = TokenIterator.instance.reset(@model.tokenizedLineForScreenRow(targetRow)) + while iterator.next() + characterWidths = @getScopedCharacterWidths(iterator.getScopes()) valueIndex = 0 - while valueIndex < token.value.length - if token.hasPairedCharacter - char = token.value.substr(valueIndex, 2) + text = iterator.getText() + while valueIndex < text.length + if iterator.isPairedCharacter() + char = text charLength = 2 valueIndex += 2 else - char = token.value[valueIndex] + char = text[valueIndex] charLength = 1 valueIndex++ From 0ca967d6b018d84ef23d104d2d5c815d3fa2d529 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Thu, 14 May 2015 01:10:29 +0200 Subject: [PATCH 30/50] Switch character measurement to TokenIterator Instead of using TokenizedLine::tokens shim --- src/lines-component.coffee | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/lines-component.coffee b/src/lines-component.coffee index 80b32b88b..8fa7c215f 100644 --- a/src/lines-component.coffee +++ b/src/lines-component.coffee @@ -354,19 +354,22 @@ class LinesComponent iterator = null charIndex = 0 - for {value, scopes, hasPairedCharacter} in tokenizedLine.tokens + tokenIterator = TokenIterator.instance.reset(tokenizedLine) + while tokenIterator.next() + scopes = tokenIterator.getScopes() + text = tokenIterator.getText() charWidths = @presenter.getScopedCharacterWidths(scopes) - valueIndex = 0 - while valueIndex < value.length - if hasPairedCharacter - char = value.substr(valueIndex, 2) + textIndex = 0 + while textIndex < text.length + if tokenIterator.isPairedCharacter() + char = text charLength = 2 - valueIndex += 2 + textIndex += 2 else - char = value[valueIndex] + char = text[textIndex] charLength = 1 - valueIndex++ + textIndex++ continue if char is '\0' From 37d9a00b375ec99ddc4fbc056a54f2ef92f0eaab Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Thu, 14 May 2015 01:15:44 +0200 Subject: [PATCH 31/50] Use TokenIterator in DisplayBuffer instead of tokens shim --- src/display-buffer.coffee | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/src/display-buffer.coffee b/src/display-buffer.coffee index f4c078b17..e44822cc1 100644 --- a/src/display-buffer.coffee +++ b/src/display-buffer.coffee @@ -2,14 +2,15 @@ _ = require 'underscore-plus' Serializable = require 'serializable' {CompositeDisposable, Emitter} = require 'event-kit' {Point, Range} = require 'text-buffer' +Grim = require 'grim' TokenizedBuffer = require './tokenized-buffer' +TokenIterator = require './token-iterator' RowMap = require './row-map' Fold = require './fold' Model = require './model' Token = require './token' Decoration = require './decoration' Marker = require './marker' -Grim = require 'grim' class BufferToScreenConversionError extends Error constructor: (@message, @metadata) -> @@ -651,16 +652,19 @@ class DisplayBuffer extends Model top = targetRow * @lineHeightInPixels left = 0 column = 0 - for token in @tokenizedLineForScreenRow(targetRow).tokens - charWidths = @getScopedCharWidths(token.scopes) + + iterator = TokenIterator.instance.reset(@tokenizedLineForScreenRow(targetRow)) + while iterator.next() + charWidths = @getScopedCharWidths(iterator.getScopes()) valueIndex = 0 - while valueIndex < token.value.length - if token.hasPairedCharacter - char = token.value.substr(valueIndex, 2) + value = iterator.getText() + while valueIndex < value.length + if iterator.isPairedCharacter() + char = value charLength = 2 valueIndex += 2 else - char = token.value[valueIndex] + char = value[valueIndex] charLength = 1 valueIndex++ @@ -681,16 +685,19 @@ class DisplayBuffer extends Model left = 0 column = 0 - for token in @tokenizedLineForScreenRow(row).tokens - charWidths = @getScopedCharWidths(token.scopes) + + iterator = TokenIterator.instance.reset(@tokenizedLineForScreenRow(row)) + while iterator.next() + charWidths = @getScopedCharWidths(iterator.getScopes()) + value = iterator.getText() valueIndex = 0 - while valueIndex < token.value.length - if token.hasPairedCharacter - char = token.value.substr(valueIndex, 2) + while valueIndex < value.length + if iterator.isPairedCharacter() + char = value charLength = 2 valueIndex += 2 else - char = token.value[valueIndex] + char = value[valueIndex] charLength = 1 valueIndex++ From 173bc82e42460fb3855762ef774a5ff991c40988 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Thu, 14 May 2015 01:46:32 +0200 Subject: [PATCH 32/50] =?UTF-8?q?Don=E2=80=99t=20include=20tokens=20in=20p?= =?UTF-8?q?resenter=20state=20for=20lines?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- spec/text-editor-presenter-spec.coffee | 36 ++++++++++++++++++++------ src/text-editor-presenter.coffee | 1 - 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/spec/text-editor-presenter-spec.coffee b/spec/text-editor-presenter-spec.coffee index de2dd780d..40dae08cc 100644 --- a/spec/text-editor-presenter-spec.coffee +++ b/spec/text-editor-presenter-spec.coffee @@ -670,7 +670,11 @@ describe "TextEditorPresenter", -> expectValues lineStateForScreenRow(presenter, 4), { screenRow: 4 text: line4.text - tokens: line4.tokens + tags: line4.tags + specialTokens: line4.specialTokens + firstNonWhitespaceIndex: line4.firstNonWhitespaceIndex + firstTrailingWhitespaceIndex: line4.firstTrailingWhitespaceIndex + invisibles: line4.invisibles top: 10 * 4 } @@ -678,7 +682,11 @@ describe "TextEditorPresenter", -> expectValues lineStateForScreenRow(presenter, 5), { screenRow: 5 text: line5.text - tokens: line5.tokens + tags: line5.tags + specialTokens: line5.specialTokens + firstNonWhitespaceIndex: line5.firstNonWhitespaceIndex + firstTrailingWhitespaceIndex: line5.firstTrailingWhitespaceIndex + invisibles: line5.invisibles top: 10 * 5 } @@ -686,7 +694,11 @@ describe "TextEditorPresenter", -> expectValues lineStateForScreenRow(presenter, 6), { screenRow: 6 text: line6.text - tokens: line6.tokens + tags: line6.tags + specialTokens: line6.specialTokens + firstNonWhitespaceIndex: line6.firstNonWhitespaceIndex + firstTrailingWhitespaceIndex: line6.firstTrailingWhitespaceIndex + invisibles: line6.invisibles top: 10 * 6 } @@ -694,7 +706,11 @@ describe "TextEditorPresenter", -> expectValues lineStateForScreenRow(presenter, 7), { screenRow: 7 text: line7.text - tokens: line7.tokens + tags: line7.tags + specialTokens: line7.specialTokens + firstNonWhitespaceIndex: line7.firstNonWhitespaceIndex + firstTrailingWhitespaceIndex: line7.firstTrailingWhitespaceIndex + invisibles: line7.invisibles top: 10 * 7 } @@ -702,7 +718,11 @@ describe "TextEditorPresenter", -> expectValues lineStateForScreenRow(presenter, 8), { screenRow: 8 text: line8.text - tokens: line8.tokens + tags: line8.tags + specialTokens: line8.specialTokens + firstNonWhitespaceIndex: line8.firstNonWhitespaceIndex + firstTrailingWhitespaceIndex: line8.firstTrailingWhitespaceIndex + invisibles: line8.invisibles top: 10 * 8 } @@ -797,19 +817,19 @@ describe "TextEditorPresenter", -> line1 = editor.tokenizedLineForScreenRow(1) expectValues lineStateForScreenRow(presenter, 1), { text: line1.text - tokens: line1.tokens + tags: line1.tags } line2 = editor.tokenizedLineForScreenRow(2) expectValues lineStateForScreenRow(presenter, 2), { text: line2.text - tokens: line2.tokens + tags: line2.tags } line3 = editor.tokenizedLineForScreenRow(3) expectValues lineStateForScreenRow(presenter, 3), { text: line3.text - tokens: line3.tokens + tags: line3.tags } it "does not remove out-of-view lines corresponding to ::mouseWheelScreenRow until ::stoppedScrollingDelay elapses", -> diff --git a/src/text-editor-presenter.coffee b/src/text-editor-presenter.coffee index 9be8e9bc0..7fb742bae 100644 --- a/src/text-editor-presenter.coffee +++ b/src/text-editor-presenter.coffee @@ -342,7 +342,6 @@ class TextEditorPresenter firstTrailingWhitespaceIndex: line.firstTrailingWhitespaceIndex invisibles: line.invisibles endOfLineInvisibles: line.endOfLineInvisibles - tokens: line.tokens isOnlyWhitespace: line.isOnlyWhitespace() indentLevel: line.indentLevel tabLength: line.tabLength From 64c0ef8bd8dd4a69197a1fb76cfab6f604c92410 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Thu, 14 May 2015 01:46:49 +0200 Subject: [PATCH 33/50] Remove more usages of tokens shim --- src/tokenized-buffer.coffee | 7 +++++- src/tokenized-line.coffee | 43 +++++++++++++++++++++---------------- 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/src/tokenized-buffer.coffee b/src/tokenized-buffer.coffee index b33fdd8fc..4ce0f44d3 100644 --- a/src/tokenized-buffer.coffee +++ b/src/tokenized-buffer.coffee @@ -5,6 +5,7 @@ _ = require 'underscore-plus' Serializable = require 'serializable' Model = require './model' TokenizedLine = require './tokenized-line' +TokenIterator = require './token-iterator' Token = require './token' ScopeDescriptor = require './scope-descriptor' Grim = require 'grim' @@ -379,7 +380,11 @@ class TokenizedBuffer extends Model 0 scopeDescriptorForPosition: (position) -> - new ScopeDescriptor(scopes: @tokenForPosition(position).scopes) + {row, column} = Point.fromObject(position) + iterator = TokenIterator.instance.reset(@tokenizedLines[row]) + while iterator.next() + break if iterator.getScreenEnd() > column + new ScopeDescriptor(scopes: iterator.getScopes().slice()) tokenForPosition: (position) -> {row, column} = Point.fromObject(position) diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index 2183ac349..3c6851eef 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -8,6 +8,7 @@ NonWhitespaceRegex = /\S/ LeadingWhitespaceRegex = /^\s*/ TrailingWhitespaceRegex = /\s*$/ RepeatedSpaceRegex = /[ ]/g +CommentScopeRegex = /(\b|\.)comment/ idCounter = 1 module.exports = @@ -201,28 +202,30 @@ class TokenizedLine # # Returns a {Number} representing the clipped column. clipScreenColumn: (column, options={}) -> - return 0 if @tokens.length is 0 + return 0 if @tags.length is 0 {clip} = options column = Math.min(column, @getMaxScreenColumn()) tokenStartColumn = 0 - for token in @tokens - break if tokenStartColumn + token.screenDelta > column - tokenStartColumn += token.screenDelta - if @isColumnInsideSoftWrapIndentation(tokenStartColumn) - @getSoftWrapIndentationDelta() - else if token.isAtomic and tokenStartColumn < column + iterator = TokenIterator.instance.reset(this) + while iterator.next() + break if iterator.getScreenEnd() > column + + if iterator.isSoftWrapIndentation() + iterator.next() while iterator.isSoftWrapIndentation() + iterator.getScreenStart() + else if iterator.isAtomic() and iterator.getScreenStart() < column if clip is 'forward' - tokenStartColumn + token.screenDelta + iterator.getScreenEnd() else if clip is 'backward' - tokenStartColumn + iterator.getScreenStart() else #'closest' - if column > tokenStartColumn + (token.screenDelta / 2) - tokenStartColumn + token.screenDelta + if column > ((iterator.getScreenStart() + iterator.getScreenEnd()) / 2) + iterator.getScreenEnd() else - tokenStartColumn + iterator.getScreenStart() else column @@ -434,11 +437,13 @@ class TokenizedLine @endOfLineInvisibles.push(eol) if eol isComment: -> - for token in @tokens - continue if token.scopes.length is 1 - continue if token.isOnlyWhitespace() - for scope in token.scopes - return true if _.contains(scope.split('.'), 'comment') + iterator = TokenIterator.instance.reset(this) + while iterator.next() + scopes = iterator.getScopes() + continue if scopes.length is 1 + continue unless NonWhitespaceRegex.test(iterator.getText()) + for scope in scopes + return true if CommentScopeRegex.test(scope) break false @@ -449,4 +454,6 @@ class TokenizedLine @tokens[index] getTokenCount: -> - @tokens.length + count = 0 + count++ for tag in @tags when tag >= 0 + count From 64f576624ec22b41d9209efdaa929a7734e52270 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Thu, 14 May 2015 17:39:23 +0200 Subject: [PATCH 34/50] Avoid tokens shim in suggestedIndent code --- src/language-mode.coffee | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/language-mode.coffee b/src/language-mode.coffee index b5529a05e..1d7def178 100644 --- a/src/language-mode.coffee +++ b/src/language-mode.coffee @@ -2,6 +2,7 @@ _ = require 'underscore-plus' {OnigRegExp} = require 'oniguruma' ScopeDescriptor = require './scope-descriptor' +TokenIterator = require './token-iterator' module.exports = class LanguageMode @@ -242,8 +243,9 @@ class LanguageMode @suggestedIndentForTokenizedLineAtBufferRow(bufferRow, tokenizedLine, options) suggestedIndentForTokenizedLineAtBufferRow: (bufferRow, tokenizedLine, options) -> - scopes = tokenizedLine.tokens[0].scopes - scopeDescriptor = new ScopeDescriptor({scopes}) + iterator = TokenIterator.instance.reset(tokenizedLine) + iterator.next() + scopeDescriptor = new ScopeDescriptor(scopes: iterator.getScopes()) currentIndentLevel = @editor.indentationForBufferRow(bufferRow) return currentIndentLevel unless increaseIndentRegex = @increaseIndentRegexForScopeDescriptor(scopeDescriptor) From 27b30303e1760a00d08c1af27bdf2829ac1a2605 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Thu, 14 May 2015 17:39:35 +0200 Subject: [PATCH 35/50] Avoid tokens shim in isBufferRowCommented --- src/text-editor.coffee | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/text-editor.coffee b/src/text-editor.coffee index ac7c7d4f2..0c4d0cbf7 100644 --- a/src/text-editor.coffee +++ b/src/text-editor.coffee @@ -2471,9 +2471,8 @@ class TextEditor extends Model # Extended: Determine if the given row is entirely a comment isBufferRowCommented: (bufferRow) -> if match = @lineTextForBufferRow(bufferRow).match(/\S/) - scopeDescriptor = @tokenForBufferPosition([bufferRow, match.index]).scopes @commentScopeSelector ?= new TextMateScopeSelector('comment.*') - @commentScopeSelector.matches(scopeDescriptor) + @commentScopeSelector.matches(@scopeDescriptorForBufferPosition([bufferRow, match.index]).scopes) logCursorScope: -> scopeDescriptor = @getLastCursor().getScopeDescriptor() From ac5a5d5ba0939296cd4509d47c8270402fce7358 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Thu, 14 May 2015 17:51:46 +0200 Subject: [PATCH 36/50] Remove unused TokenizedBuffer methods that relied on tokens shim --- spec/tokenized-buffer-spec.coffee | 8 ----- src/tokenized-buffer.coffee | 60 ------------------------------- 2 files changed, 68 deletions(-) diff --git a/spec/tokenized-buffer-spec.coffee b/spec/tokenized-buffer-spec.coffee index 0c5363ad3..c51d6106f 100644 --- a/spec/tokenized-buffer-spec.coffee +++ b/spec/tokenized-buffer-spec.coffee @@ -296,14 +296,6 @@ describe "TokenizedBuffer", -> expect(tokenizedBuffer.tokenizedLineForRow(5).ruleStack?).toBeTruthy() expect(tokenizedBuffer.tokenizedLineForRow(6).ruleStack?).toBeTruthy() - describe ".findOpeningBracket(closingBufferPosition)", -> - it "returns the position of the matching bracket, skipping any nested brackets", -> - expect(tokenizedBuffer.findOpeningBracket([9, 2])).toEqual [1, 29] - - describe ".findClosingBracket(startBufferPosition)", -> - it "returns the position of the matching bracket, skipping any nested brackets", -> - expect(tokenizedBuffer.findClosingBracket([1, 29])).toEqual [9, 2] - it "tokenizes leading whitespace based on the new tab length", -> expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[0].isAtomic).toBeTruthy() expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[0].value).toBe " " diff --git a/src/tokenized-buffer.coffee b/src/tokenized-buffer.coffee index 4ce0f44d3..e3f09b9b5 100644 --- a/src/tokenized-buffer.coffee +++ b/src/tokenized-buffer.coffee @@ -444,66 +444,6 @@ class TokenizedBuffer extends Model new Range(new Point(position.row, startColumn), new Point(position.row, endColumn)) - iterateTokensInBufferRange: (bufferRange, iterator) -> - bufferRange = Range.fromObject(bufferRange) - {start, end} = bufferRange - - keepLooping = true - stop = -> keepLooping = false - - for bufferRow in [start.row..end.row] - bufferColumn = 0 - for token in @tokenizedLines[bufferRow].tokens - startOfToken = new Point(bufferRow, bufferColumn) - iterator(token, startOfToken, {stop}) if bufferRange.containsPoint(startOfToken) - return unless keepLooping - bufferColumn += token.bufferDelta - - backwardsIterateTokensInBufferRange: (bufferRange, iterator) -> - bufferRange = Range.fromObject(bufferRange) - {start, end} = bufferRange - - keepLooping = true - stop = -> keepLooping = false - - for bufferRow in [end.row..start.row] - bufferColumn = @buffer.lineLengthForRow(bufferRow) - for token in new Array(@tokenizedLines[bufferRow].tokens...).reverse() - bufferColumn -= token.bufferDelta - startOfToken = new Point(bufferRow, bufferColumn) - iterator(token, startOfToken, {stop}) if bufferRange.containsPoint(startOfToken) - return unless keepLooping - - findOpeningBracket: (startBufferPosition) -> - range = [[0,0], startBufferPosition] - position = null - depth = 0 - @backwardsIterateTokensInBufferRange range, (token, startPosition, {stop}) -> - if token.isBracket() - if token.value is '}' - depth++ - else if token.value is '{' - depth-- - if depth is 0 - position = startPosition - stop() - position - - findClosingBracket: (startBufferPosition) -> - range = [startBufferPosition, @buffer.getEndPosition()] - position = null - depth = 0 - @iterateTokensInBufferRange range, (token, startPosition, {stop}) -> - if token.isBracket() - if token.value is '{' - depth++ - else if token.value is '}' - depth-- - if depth is 0 - position = startPosition - stop() - position - # Gets the row number of the last line. # # Returns a {Number}. From 915b13e75c50552386d821bc5ba71778109ef4a5 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Thu, 14 May 2015 21:35:51 +0200 Subject: [PATCH 37/50] :arrow_up: first-mate to 4.0.0 for tagged token representation --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c2bdb47f5..bfa032303 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "delegato": "^1", "emissary": "^1.3.3", "event-kit": "^1.1.1", - "first-mate": "^3.1", + "first-mate": "^4.0.0", "fs-plus": "^2.8.0", "fstream": "0.1.24", "fuzzaldrin": "^2.1", From 4235c15dd858bc63be8c634921e4b44014583c69 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Thu, 14 May 2015 21:43:45 +0200 Subject: [PATCH 38/50] :fire: debugger --- src/lines-component.coffee | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lines-component.coffee b/src/lines-component.coffee index 8fa7c215f..51ea65f28 100644 --- a/src/lines-component.coffee +++ b/src/lines-component.coffee @@ -177,7 +177,6 @@ class LinesComponent innerHTML = "" iterator = TokenIterator.instance.reset(lineState) - debugger if global.debug while iterator.next() for scope in iterator.getScopeEnds() innerHTML += "" From 3ea59696e6f63005ccb5e3a81814c66d1815b09e Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Thu, 14 May 2015 22:31:51 +0200 Subject: [PATCH 39/50] :arrow_up: first-mate --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1722e5e64..bf8a4edcb 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "delegato": "^1", "emissary": "^1.3.3", "event-kit": "^1.1.1", - "first-mate": "^4.0.0", + "first-mate": "^4.1.0", "fs-plus": "^2.8.0", "fstream": "0.1.24", "fuzzaldrin": "^2.1", From ec4c453b87ad7fa6db2961e93eda689d3dfa84e8 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Thu, 14 May 2015 23:08:02 +0200 Subject: [PATCH 40/50] :arrow_up: first-mate --- package.json | 2 +- src/tokenized-buffer.coffee | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index bf8a4edcb..b7b03cfa7 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "delegato": "^1", "emissary": "^1.3.3", "event-kit": "^1.1.1", - "first-mate": "^4.1.0", + "first-mate": "^4.1.1", "fs-plus": "^2.8.0", "fstream": "0.1.24", "fuzzaldrin": "^2.1", diff --git a/src/tokenized-buffer.coffee b/src/tokenized-buffer.coffee index 01631a221..03b9dcaac 100644 --- a/src/tokenized-buffer.coffee +++ b/src/tokenized-buffer.coffee @@ -330,7 +330,7 @@ class TokenizedBuffer extends Model lineEnding = @buffer.lineEndingForRow(row) tabLength = @getTabLength() indentLevel = @indentLevelForRow(row) - {tags, ruleStack} = @grammar.tokenizeLine(text, ruleStack, row is 0) + {tags, ruleStack} = @grammar.tokenizeLine(text, ruleStack, row is 0, false) new TokenizedLine({openScopes, text, tags, ruleStack, tabLength, lineEnding, indentLevel, invisibles: @getInvisiblesToShow()}) getInvisiblesToShow: -> From 4b5b64f782493e32ce414acf61e2236459ffbda3 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Sat, 16 May 2015 00:06:52 +0200 Subject: [PATCH 41/50] Return scopes at very end of token for the last token on a line --- src/tokenized-buffer.coffee | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/tokenized-buffer.coffee b/src/tokenized-buffer.coffee index 03b9dcaac..62a68202e 100644 --- a/src/tokenized-buffer.coffee +++ b/src/tokenized-buffer.coffee @@ -401,10 +401,14 @@ class TokenizedBuffer extends Model scopeDescriptorForPosition: (position) -> {row, column} = Point.fromObject(position) + positionIsEndOfLine = column is @tokenizedLines[row].text.length + iterator = TokenIterator.instance.reset(@tokenizedLines[row]) while iterator.next() - break if iterator.getScreenEnd() > column - new ScopeDescriptor(scopes: iterator.getScopes().slice()) + tokenEnd = iterator.getScreenEnd() + break if tokenEnd > column or (positionIsEndOfLine and tokenEnd is column) + + new ScopeDescriptor(scopes: iterator.getScopes()) tokenForPosition: (position) -> {row, column} = Point.fromObject(position) From 1923033198a857d248381046a2dec72e304493ad Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Sat, 16 May 2015 00:12:43 +0200 Subject: [PATCH 42/50] Fix spurious lint error by rearranging syntax --- src/tokenized-line.coffee | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index 3c6851eef..e7ed27406 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -321,7 +321,8 @@ class TokenizedLine leftTags.push(leftSuffix) if leftSuffix > 0 softWrapIndent = @indentLevel * @tabLength + (hangingIndent ? 0) - rightText = ' ' + rightText for i in [0...softWrapIndent] by 1 + for i in [0...softWrapIndent] by 1 + rightText = ' ' + rightText remainingSoftWrapIndent = softWrapIndent while remainingSoftWrapIndent > 0 indentToken = Math.min(remainingSoftWrapIndent, @tabLength) From 0c153a298e386e36a56cd21b0b42d946daea7ca3 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Sat, 16 May 2015 00:46:31 +0200 Subject: [PATCH 43/50] Use scope of last token if we iterate off the end --- src/tokenized-buffer.coffee | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/tokenized-buffer.coffee b/src/tokenized-buffer.coffee index 62a68202e..3baba3b72 100644 --- a/src/tokenized-buffer.coffee +++ b/src/tokenized-buffer.coffee @@ -401,14 +401,19 @@ class TokenizedBuffer extends Model scopeDescriptorForPosition: (position) -> {row, column} = Point.fromObject(position) - positionIsEndOfLine = column is @tokenizedLines[row].text.length iterator = TokenIterator.instance.reset(@tokenizedLines[row]) while iterator.next() - tokenEnd = iterator.getScreenEnd() - break if tokenEnd > column or (positionIsEndOfLine and tokenEnd is column) + if iterator.getScreenEnd() > column + scopes = iterator.getScopes() + break - new ScopeDescriptor(scopes: iterator.getScopes()) + # rebuild scope of last token if we iterated off the end + unless scopes? + scopes = iterator.getScopes() + scopes.push(iterator.getScopeEnds().reverse()...) + + new ScopeDescriptor({scopes}) tokenForPosition: (position) -> {row, column} = Point.fromObject(position) From 66d9b30f990aafef47d2d45c95fb40e78534e351 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Sat, 16 May 2015 02:15:42 +0200 Subject: [PATCH 44/50] :arrow_up: first-mate --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b7b03cfa7..d3c6f86f3 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "delegato": "^1", "emissary": "^1.3.3", "event-kit": "^1.1.1", - "first-mate": "^4.1.1", + "first-mate": "^4.1.2", "fs-plus": "^2.8.0", "fstream": "0.1.24", "fuzzaldrin": "^2.1", From 4ddbb01dfaee604e4547e3881c88dbff1f77a4a7 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Sat, 16 May 2015 03:03:13 +0200 Subject: [PATCH 45/50] :arrow_up: language-shellscript for spec fixes against first-mate 4.x --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d3c6f86f3..7851e37a4 100644 --- a/package.json +++ b/package.json @@ -151,7 +151,7 @@ "language-ruby": "0.52.0", "language-ruby-on-rails": "0.21.0", "language-sass": "0.38.0", - "language-shellscript": "0.14.0", + "language-shellscript": "0.15.0", "language-source": "0.9.0", "language-sql": "0.15.0", "language-text": "0.6.0", From 458b727fe0347d36dcc3f14d466341b2e3c69ca8 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Tue, 19 May 2015 01:51:53 +0200 Subject: [PATCH 46/50] :arrow_up: first-mate --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ba3a2e20b..d248773a6 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "delegato": "^1", "emissary": "^1.3.3", "event-kit": "^1.1.1", - "first-mate": "^4.1.2", + "first-mate": "^4.1.3", "fs-plus": "^2.8.0", "fstream": "0.1.24", "fuzzaldrin": "^2.1", From 564d75c534b5cf1fb0176977ac6687564048b10e Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Tue, 19 May 2015 17:22:25 +0200 Subject: [PATCH 47/50] :arrow_up: first-mate --- package.json | 2 +- src/tokenized-buffer.coffee | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index f5d344ea5..912fe0ede 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "delegato": "^1", "emissary": "^1.3.3", "event-kit": "^1.1.1", - "first-mate": "^4.1.3", + "first-mate": "^4.1.4", "fs-plus": "^2.8.0", "fstream": "0.1.24", "fuzzaldrin": "^2.1", diff --git a/src/tokenized-buffer.coffee b/src/tokenized-buffer.coffee index 3baba3b72..a709d9755 100644 --- a/src/tokenized-buffer.coffee +++ b/src/tokenized-buffer.coffee @@ -315,7 +315,7 @@ class TokenizedBuffer extends Model @buildPlaceholderTokenizedLineForRow(row) for row in [startRow..endRow] buildPlaceholderTokenizedLineForRow: (row) -> - openScopes = [@grammar.idForScope(@grammar.scopeName)] + openScopes = [@grammar.startIdForScope(@grammar.scopeName)] text = @buffer.lineForRow(row) tags = [text.length] tabLength = @getTabLength() From 13db572ac84a0979294bd3dacde8ce7f0c410713 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Tue, 19 May 2015 17:55:36 +0200 Subject: [PATCH 48/50] Avoid error in donna npm while generating documentation --- src/token-iterator.coffee | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/token-iterator.coffee b/src/token-iterator.coffee index 800f76d03..65a4c3100 100644 --- a/src/token-iterator.coffee +++ b/src/token-iterator.coffee @@ -2,8 +2,6 @@ module.exports = class TokenIterator - @instance: new this - constructor: (line) -> @reset(line) if line? @@ -83,3 +81,5 @@ class TokenIterator isAtomic: -> @isSoftTab() or @isHardTab() or @isSoftWrapIndentation() or @isPairedCharacter() + +TokenIterator.instance = new TokenIterator From a109b3811c61f6115a398a757f09feae648f3eb7 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Tue, 19 May 2015 23:19:45 +0200 Subject: [PATCH 49/50] Add TokenizedLine::getTokenIterator This will not be part of the public API but will replace another non-public API usage in autocomplete-plus. --- src/tokenized-line.coffee | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index e7ed27406..acaa16378 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -143,8 +143,10 @@ class TokenizedLine @lineIsWhitespaceOnly = true @firstTrailingWhitespaceIndex = 0 + getTokenIterator: -> TokenIterator.instance.reset(this) + Object.defineProperty @prototype, 'tokens', get: -> - iterator = TokenIterator.instance.reset(this) + iterator = @getTokenIterator() tokens = [] while iterator.next() @@ -209,7 +211,7 @@ class TokenizedLine tokenStartColumn = 0 - iterator = TokenIterator.instance.reset(this) + iterator = @getTokenIterator() while iterator.next() break if iterator.getScreenEnd() > column @@ -230,7 +232,7 @@ class TokenizedLine column screenColumnForBufferColumn: (targetBufferColumn, options) -> - iterator = TokenIterator.instance.reset(this) + iterator = @getTokenIterator() while iterator.next() tokenBufferStart = iterator.getBufferStart() tokenBufferEnd = iterator.getBufferEnd() @@ -243,7 +245,7 @@ class TokenizedLine iterator.getScreenEnd() bufferColumnForScreenColumn: (targetScreenColumn) -> - iterator = TokenIterator.instance.reset(this) + iterator = @getTokenIterator() while iterator.next() tokenScreenStart = iterator.getScreenStart() tokenScreenEnd = iterator.getScreenEnd() @@ -438,7 +440,7 @@ class TokenizedLine @endOfLineInvisibles.push(eol) if eol isComment: -> - iterator = TokenIterator.instance.reset(this) + iterator = @getTokenIterator() while iterator.next() scopes = iterator.getScopes() continue if scopes.length is 1 From 2beb6c0fe019de8d490e65bc27eb8346bec5752b Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Wed, 20 May 2015 19:12:22 +0200 Subject: [PATCH 50/50] Pass TokenIterator reference instead of using global singleton Adds an extra reference to each tokenized line but is also more sane. /cc @maxbrunsfeld --- src/display-buffer.coffee | 5 ++--- src/language-mode.coffee | 3 +-- src/lines-component.coffee | 31 ++++++++++++++++--------------- src/text-editor-presenter.coffee | 3 +-- src/token-iterator.coffee | 2 -- src/tokenized-buffer.coffee | 7 ++++--- src/tokenized-line.coffee | 8 +++++--- 7 files changed, 29 insertions(+), 30 deletions(-) diff --git a/src/display-buffer.coffee b/src/display-buffer.coffee index 57fc64759..b156bf98a 100644 --- a/src/display-buffer.coffee +++ b/src/display-buffer.coffee @@ -4,7 +4,6 @@ Serializable = require 'serializable' {Point, Range} = require 'text-buffer' Grim = require 'grim' TokenizedBuffer = require './tokenized-buffer' -TokenIterator = require './token-iterator' RowMap = require './row-map' Fold = require './fold' Model = require './model' @@ -652,7 +651,7 @@ class DisplayBuffer extends Model left = 0 column = 0 - iterator = TokenIterator.instance.reset(@tokenizedLineForScreenRow(targetRow)) + iterator = @tokenizedLineForScreenRow(targetRow).getTokenIterator() while iterator.next() charWidths = @getScopedCharWidths(iterator.getScopes()) valueIndex = 0 @@ -685,7 +684,7 @@ class DisplayBuffer extends Model left = 0 column = 0 - iterator = TokenIterator.instance.reset(@tokenizedLineForScreenRow(row)) + iterator = @tokenizedLineForScreenRow(row).getTokenIterator() while iterator.next() charWidths = @getScopedCharWidths(iterator.getScopes()) value = iterator.getText() diff --git a/src/language-mode.coffee b/src/language-mode.coffee index 1d7def178..c9401550b 100644 --- a/src/language-mode.coffee +++ b/src/language-mode.coffee @@ -2,7 +2,6 @@ _ = require 'underscore-plus' {OnigRegExp} = require 'oniguruma' ScopeDescriptor = require './scope-descriptor' -TokenIterator = require './token-iterator' module.exports = class LanguageMode @@ -243,7 +242,7 @@ class LanguageMode @suggestedIndentForTokenizedLineAtBufferRow(bufferRow, tokenizedLine, options) suggestedIndentForTokenizedLineAtBufferRow: (bufferRow, tokenizedLine, options) -> - iterator = TokenIterator.instance.reset(tokenizedLine) + iterator = tokenizedLine.getTokenIterator() iterator.next() scopeDescriptor = new ScopeDescriptor(scopes: iterator.getScopes()) diff --git a/src/lines-component.coffee b/src/lines-component.coffee index 51ea65f28..17c904e99 100644 --- a/src/lines-component.coffee +++ b/src/lines-component.coffee @@ -22,6 +22,7 @@ class LinesComponent placeholderTextDiv: null constructor: ({@presenter, @hostElement, @useShadowDOM, visible}) -> + @tokenIterator = new TokenIterator @measuredLines = new Set @lineNodesByLineId = {} @screenRowsByLineId = {} @@ -175,19 +176,19 @@ class LinesComponent lineIsWhitespaceOnly = firstTrailingWhitespaceIndex is 0 innerHTML = "" - iterator = TokenIterator.instance.reset(lineState) + @tokenIterator.reset(lineState) - while iterator.next() - for scope in iterator.getScopeEnds() + while @tokenIterator.next() + for scope in @tokenIterator.getScopeEnds() innerHTML += "" - for scope in iterator.getScopeStarts() + for scope in @tokenIterator.getScopeStarts() innerHTML += "" - tokenStart = iterator.getScreenStart() - tokenEnd = iterator.getScreenEnd() - tokenText = iterator.getText() - isHardTab = iterator.isHardTab() + tokenStart = @tokenIterator.getScreenStart() + tokenEnd = @tokenIterator.getScreenEnd() + tokenText = @tokenIterator.getText() + isHardTab = @tokenIterator.isHardTab() if hasLeadingWhitespace = tokenStart < firstNonWhitespaceIndex tokenFirstNonWhitespaceIndex = firstNonWhitespaceIndex - tokenStart @@ -209,10 +210,10 @@ class LinesComponent innerHTML += @buildTokenHTML(tokenText, isHardTab, tokenFirstNonWhitespaceIndex, tokenFirstTrailingWhitespaceIndex, hasIndentGuide, hasInvisibleCharacters) - for scope in iterator.getScopeEnds() + for scope in @tokenIterator.getScopeEnds() innerHTML += "" - for scope in iterator.getScopes() + for scope in @tokenIterator.getScopes() innerHTML += "" innerHTML += @buildEndOfLineHTML(id) @@ -353,15 +354,15 @@ class LinesComponent iterator = null charIndex = 0 - tokenIterator = TokenIterator.instance.reset(tokenizedLine) - while tokenIterator.next() - scopes = tokenIterator.getScopes() - text = tokenIterator.getText() + @tokenIterator.reset(tokenizedLine) + while @tokenIterator.next() + scopes = @tokenIterator.getScopes() + text = @tokenIterator.getText() charWidths = @presenter.getScopedCharacterWidths(scopes) textIndex = 0 while textIndex < text.length - if tokenIterator.isPairedCharacter() + if @tokenIterator.isPairedCharacter() char = text charLength = 2 textIndex += 2 diff --git a/src/text-editor-presenter.coffee b/src/text-editor-presenter.coffee index 15241625a..3aea57f29 100644 --- a/src/text-editor-presenter.coffee +++ b/src/text-editor-presenter.coffee @@ -2,7 +2,6 @@ {Point, Range} = require 'text-buffer' _ = require 'underscore-plus' Decoration = require './decoration' -TokenIterator = require './token-iterator' module.exports = class TextEditorPresenter @@ -1013,7 +1012,7 @@ class TextEditorPresenter left = 0 column = 0 - iterator = TokenIterator.instance.reset(@model.tokenizedLineForScreenRow(targetRow)) + iterator = @model.tokenizedLineForScreenRow(targetRow).getTokenIterator() while iterator.next() characterWidths = @getScopedCharacterWidths(iterator.getScopes()) diff --git a/src/token-iterator.coffee b/src/token-iterator.coffee index 65a4c3100..202b044ba 100644 --- a/src/token-iterator.coffee +++ b/src/token-iterator.coffee @@ -81,5 +81,3 @@ class TokenIterator isAtomic: -> @isSoftTab() or @isHardTab() or @isSoftWrapIndentation() or @isPairedCharacter() - -TokenIterator.instance = new TokenIterator diff --git a/src/tokenized-buffer.coffee b/src/tokenized-buffer.coffee index a709d9755..60ebe16f0 100644 --- a/src/tokenized-buffer.coffee +++ b/src/tokenized-buffer.coffee @@ -27,6 +27,7 @@ class TokenizedBuffer extends Model constructor: ({@buffer, @tabLength, @ignoreInvisibles}) -> @emitter = new Emitter @disposables = new CompositeDisposable + @tokenIterator = new TokenIterator @disposables.add atom.grammars.onDidAddGrammar(@grammarAddedOrUpdated) @disposables.add atom.grammars.onDidUpdateGrammar(@grammarAddedOrUpdated) @@ -321,7 +322,7 @@ class TokenizedBuffer extends Model tabLength = @getTabLength() indentLevel = @indentLevelForRow(row) lineEnding = @buffer.lineEndingForRow(row) - new TokenizedLine({openScopes, text, tags, tabLength, indentLevel, invisibles: @getInvisiblesToShow(), lineEnding}) + new TokenizedLine({openScopes, text, tags, tabLength, indentLevel, invisibles: @getInvisiblesToShow(), lineEnding, @tokenIterator}) buildTokenizedLineForRow: (row, ruleStack, openScopes) -> @buildTokenizedLineForRowWithText(row, @buffer.lineForRow(row), ruleStack, openScopes) @@ -331,7 +332,7 @@ class TokenizedBuffer extends Model tabLength = @getTabLength() indentLevel = @indentLevelForRow(row) {tags, ruleStack} = @grammar.tokenizeLine(text, ruleStack, row is 0, false) - new TokenizedLine({openScopes, text, tags, ruleStack, tabLength, lineEnding, indentLevel, invisibles: @getInvisiblesToShow()}) + new TokenizedLine({openScopes, text, tags, ruleStack, tabLength, lineEnding, indentLevel, invisibles: @getInvisiblesToShow(), @tokenIterator}) getInvisiblesToShow: -> if @configSettings.showInvisibles and not @ignoreInvisibles @@ -402,7 +403,7 @@ class TokenizedBuffer extends Model scopeDescriptorForPosition: (position) -> {row, column} = Point.fromObject(position) - iterator = TokenIterator.instance.reset(@tokenizedLines[row]) + iterator = @tokenizedLines[row].getTokenIterator() while iterator.next() if iterator.getScreenEnd() > column scopes = iterator.getScopes() diff --git a/src/tokenized-line.coffee b/src/tokenized-line.coffee index acaa16378..45af81e57 100644 --- a/src/tokenized-line.coffee +++ b/src/tokenized-line.coffee @@ -1,7 +1,6 @@ _ = require 'underscore-plus' {isPairedCharacter} = require './text-utils' Token = require './token' -TokenIterator = require './token-iterator' {SoftTab, HardTab, PairedCharacter, SoftWrapIndent} = require './special-token-symbols' NonWhitespaceRegex = /\S/ @@ -24,7 +23,7 @@ class TokenizedLine return unless properties? @specialTokens = {} - {@openScopes, @text, @tags, @lineEnding, @ruleStack} = properties + {@openScopes, @text, @tags, @lineEnding, @ruleStack, @tokenIterator} = properties {@startBufferColumn, @fold, @tabLength, @indentLevel, @invisibles} = properties @startBufferColumn ?= 0 @@ -143,7 +142,7 @@ class TokenizedLine @lineIsWhitespaceOnly = true @firstTrailingWhitespaceIndex = 0 - getTokenIterator: -> TokenIterator.instance.reset(this) + getTokenIterator: -> @tokenIterator.reset(this) Object.defineProperty @prototype, 'tokens', get: -> iterator = @getTokenIterator() @@ -179,6 +178,7 @@ class TokenizedLine copy: -> copy = new TokenizedLine + copy.tokenIterator = @tokenIterator copy.indentLevel = @indentLevel copy.openScopes = @openScopes copy.text = @text @@ -359,6 +359,7 @@ class TokenizedLine splitBufferColumn = @bufferColumnForScreenColumn(column) leftFragment = new TokenizedLine + leftFragment.tokenIterator = @tokenIterator leftFragment.openScopes = @openScopes leftFragment.text = leftText leftFragment.tags = leftTags @@ -374,6 +375,7 @@ class TokenizedLine leftFragment.firstTrailingWhitespaceIndex = Math.min(column, @firstTrailingWhitespaceIndex) rightFragment = new TokenizedLine + rightFragment.tokenIterator = @tokenIterator rightFragment.openScopes = rightOpenScopes rightFragment.text = rightText rightFragment.tags = rightTags