Revert "Merge pull request #6757 from atom/ns-less-memory-for-tokens"

This reverts commit 0cd1f110b5, reversing
changes made to d75d202d33.

Conflicts:
	package.json

spec/text-editor-presenter-spec.coffee

@@ -670,11 +670,7 @@ describe "TextEditorPresenter", ->
           expectValues lineStateForScreenRow(presenter, 4), {
             screenRow: 4
             text: line4.text
-            tags: line4.tags
-            specialTokens: line4.specialTokens
-            firstNonWhitespaceIndex: line4.firstNonWhitespaceIndex
-            firstTrailingWhitespaceIndex: line4.firstTrailingWhitespaceIndex
-            invisibles: line4.invisibles
+            tokens: line4.tokens
             top: 10 * 4
           }
 
@@ -682,11 +678,7 @@ describe "TextEditorPresenter", ->
           expectValues lineStateForScreenRow(presenter, 5), {
             screenRow: 5
             text: line5.text
-            tags: line5.tags
-            specialTokens: line5.specialTokens
-            firstNonWhitespaceIndex: line5.firstNonWhitespaceIndex
-            firstTrailingWhitespaceIndex: line5.firstTrailingWhitespaceIndex
-            invisibles: line5.invisibles
+            tokens: line5.tokens
             top: 10 * 5
           }
 
@@ -694,11 +686,7 @@ describe "TextEditorPresenter", ->
           expectValues lineStateForScreenRow(presenter, 6), {
             screenRow: 6
             text: line6.text
-            tags: line6.tags
-            specialTokens: line6.specialTokens
-            firstNonWhitespaceIndex: line6.firstNonWhitespaceIndex
-            firstTrailingWhitespaceIndex: line6.firstTrailingWhitespaceIndex
-            invisibles: line6.invisibles
+            tokens: line6.tokens
             top: 10 * 6
           }
 
@@ -706,11 +694,7 @@ describe "TextEditorPresenter", ->
           expectValues lineStateForScreenRow(presenter, 7), {
             screenRow: 7
             text: line7.text
-            tags: line7.tags
-            specialTokens: line7.specialTokens
-            firstNonWhitespaceIndex: line7.firstNonWhitespaceIndex
-            firstTrailingWhitespaceIndex: line7.firstTrailingWhitespaceIndex
-            invisibles: line7.invisibles
+            tokens: line7.tokens
             top: 10 * 7
           }
 
@@ -718,11 +702,7 @@ describe "TextEditorPresenter", ->
           expectValues lineStateForScreenRow(presenter, 8), {
             screenRow: 8
             text: line8.text
-            tags: line8.tags
-            specialTokens: line8.specialTokens
-            firstNonWhitespaceIndex: line8.firstNonWhitespaceIndex
-            firstTrailingWhitespaceIndex: line8.firstTrailingWhitespaceIndex
-            invisibles: line8.invisibles
+            tokens: line8.tokens
             top: 10 * 8
           }
 
@@ -817,19 +797,19 @@ describe "TextEditorPresenter", ->
           line1 = editor.tokenizedLineForScreenRow(1)
           expectValues lineStateForScreenRow(presenter, 1), {
             text: line1.text
-            tags: line1.tags
+            tokens: line1.tokens
           }
 
           line2 = editor.tokenizedLineForScreenRow(2)
           expectValues lineStateForScreenRow(presenter, 2), {
             text: line2.text
-            tags: line2.tags
+            tokens: line2.tokens
           }
 
           line3 = editor.tokenizedLineForScreenRow(3)
           expectValues lineStateForScreenRow(presenter, 3), {
             text: line3.text
-            tags: line3.tags
+            tokens: line3.tokens
           }
 
         it "does not remove out-of-view lines corresponding to ::mouseWheelScreenRow until ::stoppedScrollingDelay elapses", ->

spec/text-editor-spec.coffee

@@ -4110,9 +4110,8 @@ describe "TextEditor", ->
 
       runs ->
         grammar = atom.grammars.selectGrammar("text.js")
-        {line, tags} = grammar.tokenizeLine("var i; // http://github.com")
+        {tokens} = grammar.tokenizeLine("var i; // http://github.com")
 
-        tokens = atom.grammars.decodeTokens(line, tags)
         expect(tokens[0].value).toBe "var"
         expect(tokens[0].scopes).toEqual ["source.js", "storage.modifier.js"]
 

spec/tokenized-buffer-spec.coffee

@@ -296,6 +296,14 @@ describe "TokenizedBuffer", ->
           expect(tokenizedBuffer.tokenizedLineForRow(5).ruleStack?).toBeTruthy()
           expect(tokenizedBuffer.tokenizedLineForRow(6).ruleStack?).toBeTruthy()
 
+      describe ".findOpeningBracket(closingBufferPosition)", ->
+        it "returns the position of the matching bracket, skipping any nested brackets", ->
+          expect(tokenizedBuffer.findOpeningBracket([9, 2])).toEqual [1, 29]
+
+      describe ".findClosingBracket(startBufferPosition)", ->
+        it "returns the position of the matching bracket, skipping any nested brackets", ->
+          expect(tokenizedBuffer.findClosingBracket([1, 29])).toEqual [9, 2]
+
       it "tokenizes leading whitespace based on the new tab length", ->
         expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[0].isAtomic).toBeTruthy()
         expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[0].value).toBe "  "
@@ -572,7 +580,7 @@ describe "TokenizedBuffer", ->
 
     describe "when the selector matches a run of multiple tokens at the position", ->
       it "returns the range covered by all contigous tokens (within a single line)", ->
-        expect(tokenizedBuffer.bufferRangeForScopeAtPosition('.meta.function', [1, 18])).toEqual [[1, 6], [1, 28]]
+        expect(tokenizedBuffer.bufferRangeForScopeAtPosition('.function', [1, 18])).toEqual [[1, 6], [1, 28]]
 
   describe "when the editor.tabLength config value changes", ->
     it "updates the tab length of the tokenized lines", ->
@@ -689,6 +697,22 @@ describe "TokenizedBuffer", ->
       expect(line.tokens[0].firstNonWhitespaceIndex).toBe 2
       expect(line.tokens[line.tokens.length - 1].firstTrailingWhitespaceIndex).toBe 0
 
+    it "sets the ::firstNonWhitespaceIndex and ::firstTrailingWhitespaceIndex correctly when tokens are split for soft-wrapping", ->
+      atom.config.set("editor.showInvisibles", true)
+      atom.config.set("editor.invisibles", space: 'S')
+      buffer.setText(" token ")
+      fullyTokenize(tokenizedBuffer)
+      token = tokenizedBuffer.tokenizedLines[0].tokens[0]
+
+      [leftToken, rightToken] = token.splitAt(1)
+      expect(leftToken.hasInvisibleCharacters).toBe true
+      expect(leftToken.firstNonWhitespaceIndex).toBe 1
+      expect(leftToken.firstTrailingWhitespaceIndex).toBe null
+
+      expect(leftToken.hasInvisibleCharacters).toBe true
+      expect(rightToken.firstNonWhitespaceIndex).toBe null
+      expect(rightToken.firstTrailingWhitespaceIndex).toBe 5
+
   describe ".indentLevel on tokenized lines", ->
     beforeEach ->
       buffer = atom.project.bufferForPathSync('sample.js')
@@ -728,7 +752,7 @@ describe "TokenizedBuffer", ->
       it "updates empty line indent guides when the empty line is the last line", ->
         buffer.insert([12, 2], '\n')
 
-        # The newline and the tab need to be in two different operations to surface the bug
+        # The newline and he tab need to be in two different operations to surface the bug
         buffer.insert([12, 0], '  ')
         expect(tokenizedBuffer.tokenizedLineForRow(13).indentLevel).toBe 1
 

spec/tokenized-line-spec.coffee

@@ -17,3 +17,24 @@ describe "TokenizedLine", ->
     it "returns false when the line is not only whitespace", ->
       expect(editor.tokenizedLineForScreenRow(0).isOnlyWhitespace()).toBe false
       expect(editor.tokenizedLineForScreenRow(2).isOnlyWhitespace()).toBe false
+
+  describe "::getScopeTree()", ->
+    it "returns a tree whose inner nodes are scopeDescriptor and whose leaf nodes are tokens in those scopeDescriptor", ->
+      [tokens, tokenIndex] = []
+
+      ensureValidScopeTree = (scopeTree, scopeDescriptor=[]) ->
+        if scopeTree.children?
+          for child in scopeTree.children
+            ensureValidScopeTree(child, scopeDescriptor.concat([scopeTree.scope]))
+        else
+          expect(scopeTree).toBe tokens[tokenIndex++]
+          expect(scopeDescriptor).toEqual scopeTree.scopes
+
+      waitsForPromise ->
+        atom.project.open('coffee.coffee').then (o) -> editor = o
+
+      runs ->
+        tokenIndex = 0
+        tokens = editor.tokenizedLineForScreenRow(1).tokens
+        scopeTree = editor.tokenizedLineForScreenRow(1).getScopeTree()
+        ensureValidScopeTree(scopeTree)