Merge pull request #6757 from atom/ns-less-memory-for-tokens

Use 70% less memory for TokenizedLines by using a different representation

spec/text-editor-presenter-spec.coffee

@@ -670,7 +670,11 @@ describe "TextEditorPresenter", ->
           expectValues lineStateForScreenRow(presenter, 4), {
             screenRow: 4
             text: line4.text
-            tokens: line4.tokens
+            tags: line4.tags
+            specialTokens: line4.specialTokens
+            firstNonWhitespaceIndex: line4.firstNonWhitespaceIndex
+            firstTrailingWhitespaceIndex: line4.firstTrailingWhitespaceIndex
+            invisibles: line4.invisibles
             top: 10 * 4
           }
 
@@ -678,7 +682,11 @@ describe "TextEditorPresenter", ->
           expectValues lineStateForScreenRow(presenter, 5), {
             screenRow: 5
             text: line5.text
-            tokens: line5.tokens
+            tags: line5.tags
+            specialTokens: line5.specialTokens
+            firstNonWhitespaceIndex: line5.firstNonWhitespaceIndex
+            firstTrailingWhitespaceIndex: line5.firstTrailingWhitespaceIndex
+            invisibles: line5.invisibles
             top: 10 * 5
           }
 
@@ -686,7 +694,11 @@ describe "TextEditorPresenter", ->
           expectValues lineStateForScreenRow(presenter, 6), {
             screenRow: 6
             text: line6.text
-            tokens: line6.tokens
+            tags: line6.tags
+            specialTokens: line6.specialTokens
+            firstNonWhitespaceIndex: line6.firstNonWhitespaceIndex
+            firstTrailingWhitespaceIndex: line6.firstTrailingWhitespaceIndex
+            invisibles: line6.invisibles
             top: 10 * 6
           }
 
@@ -694,7 +706,11 @@ describe "TextEditorPresenter", ->
           expectValues lineStateForScreenRow(presenter, 7), {
             screenRow: 7
             text: line7.text
-            tokens: line7.tokens
+            tags: line7.tags
+            specialTokens: line7.specialTokens
+            firstNonWhitespaceIndex: line7.firstNonWhitespaceIndex
+            firstTrailingWhitespaceIndex: line7.firstTrailingWhitespaceIndex
+            invisibles: line7.invisibles
             top: 10 * 7
           }
 
@@ -702,7 +718,11 @@ describe "TextEditorPresenter", ->
           expectValues lineStateForScreenRow(presenter, 8), {
             screenRow: 8
             text: line8.text
-            tokens: line8.tokens
+            tags: line8.tags
+            specialTokens: line8.specialTokens
+            firstNonWhitespaceIndex: line8.firstNonWhitespaceIndex
+            firstTrailingWhitespaceIndex: line8.firstTrailingWhitespaceIndex
+            invisibles: line8.invisibles
             top: 10 * 8
           }
 
@@ -797,19 +817,19 @@ describe "TextEditorPresenter", ->
           line1 = editor.tokenizedLineForScreenRow(1)
           expectValues lineStateForScreenRow(presenter, 1), {
             text: line1.text
-            tokens: line1.tokens
+            tags: line1.tags
           }
 
           line2 = editor.tokenizedLineForScreenRow(2)
           expectValues lineStateForScreenRow(presenter, 2), {
             text: line2.text
-            tokens: line2.tokens
+            tags: line2.tags
           }
 
           line3 = editor.tokenizedLineForScreenRow(3)
           expectValues lineStateForScreenRow(presenter, 3), {
             text: line3.text
-            tokens: line3.tokens
+            tags: line3.tags
           }
 
         it "does not remove out-of-view lines corresponding to ::mouseWheelScreenRow until ::stoppedScrollingDelay elapses", ->

spec/text-editor-spec.coffee

@@ -4110,8 +4110,9 @@ describe "TextEditor", ->
 
       runs ->
         grammar = atom.grammars.selectGrammar("text.js")
-        {tokens} = grammar.tokenizeLine("var i; // http://github.com")
+        {line, tags} = grammar.tokenizeLine("var i; // http://github.com")
 
+        tokens = atom.grammars.decodeTokens(line, tags)
         expect(tokens[0].value).toBe "var"
         expect(tokens[0].scopes).toEqual ["source.js", "storage.modifier.js"]
 

spec/tokenized-buffer-spec.coffee

@@ -296,14 +296,6 @@ describe "TokenizedBuffer", ->
           expect(tokenizedBuffer.tokenizedLineForRow(5).ruleStack?).toBeTruthy()
           expect(tokenizedBuffer.tokenizedLineForRow(6).ruleStack?).toBeTruthy()
 
-      describe ".findOpeningBracket(closingBufferPosition)", ->
-        it "returns the position of the matching bracket, skipping any nested brackets", ->
-          expect(tokenizedBuffer.findOpeningBracket([9, 2])).toEqual [1, 29]
-
-      describe ".findClosingBracket(startBufferPosition)", ->
-        it "returns the position of the matching bracket, skipping any nested brackets", ->
-          expect(tokenizedBuffer.findClosingBracket([1, 29])).toEqual [9, 2]
-
       it "tokenizes leading whitespace based on the new tab length", ->
         expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[0].isAtomic).toBeTruthy()
         expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[0].value).toBe "  "
@@ -580,7 +572,7 @@ describe "TokenizedBuffer", ->
 
     describe "when the selector matches a run of multiple tokens at the position", ->
       it "returns the range covered by all contigous tokens (within a single line)", ->
-        expect(tokenizedBuffer.bufferRangeForScopeAtPosition('.function', [1, 18])).toEqual [[1, 6], [1, 28]]
+        expect(tokenizedBuffer.bufferRangeForScopeAtPosition('.meta.function', [1, 18])).toEqual [[1, 6], [1, 28]]
 
   describe "when the editor.tabLength config value changes", ->
     it "updates the tab length of the tokenized lines", ->
@@ -697,22 +689,6 @@ describe "TokenizedBuffer", ->
       expect(line.tokens[0].firstNonWhitespaceIndex).toBe 2
       expect(line.tokens[line.tokens.length - 1].firstTrailingWhitespaceIndex).toBe 0
 
-    it "sets the ::firstNonWhitespaceIndex and ::firstTrailingWhitespaceIndex correctly when tokens are split for soft-wrapping", ->
-      atom.config.set("editor.showInvisibles", true)
-      atom.config.set("editor.invisibles", space: 'S')
-      buffer.setText(" token ")
-      fullyTokenize(tokenizedBuffer)
-      token = tokenizedBuffer.tokenizedLines[0].tokens[0]
-
-      [leftToken, rightToken] = token.splitAt(1)
-      expect(leftToken.hasInvisibleCharacters).toBe true
-      expect(leftToken.firstNonWhitespaceIndex).toBe 1
-      expect(leftToken.firstTrailingWhitespaceIndex).toBe null
-
-      expect(leftToken.hasInvisibleCharacters).toBe true
-      expect(rightToken.firstNonWhitespaceIndex).toBe null
-      expect(rightToken.firstTrailingWhitespaceIndex).toBe 5
-
   describe ".indentLevel on tokenized lines", ->
     beforeEach ->
       buffer = atom.project.bufferForPathSync('sample.js')
@@ -752,7 +728,7 @@ describe "TokenizedBuffer", ->
       it "updates empty line indent guides when the empty line is the last line", ->
         buffer.insert([12, 2], '\n')
 
-        # The newline and he tab need to be in two different operations to surface the bug
+        # The newline and the tab need to be in two different operations to surface the bug
         buffer.insert([12, 0], '  ')
         expect(tokenizedBuffer.tokenizedLineForRow(13).indentLevel).toBe 1
 

spec/tokenized-line-spec.coffee

@@ -17,24 +17,3 @@ describe "TokenizedLine", ->
     it "returns false when the line is not only whitespace", ->
       expect(editor.tokenizedLineForScreenRow(0).isOnlyWhitespace()).toBe false
       expect(editor.tokenizedLineForScreenRow(2).isOnlyWhitespace()).toBe false
-
-  describe "::getScopeTree()", ->
-    it "returns a tree whose inner nodes are scopeDescriptor and whose leaf nodes are tokens in those scopeDescriptor", ->
-      [tokens, tokenIndex] = []
-
-      ensureValidScopeTree = (scopeTree, scopeDescriptor=[]) ->
-        if scopeTree.children?
-          for child in scopeTree.children
-            ensureValidScopeTree(child, scopeDescriptor.concat([scopeTree.scope]))
-        else
-          expect(scopeTree).toBe tokens[tokenIndex++]
-          expect(scopeDescriptor).toEqual scopeTree.scopes
-
-      waitsForPromise ->
-        atom.project.open('coffee.coffee').then (o) -> editor = o
-
-      runs ->
-        tokenIndex = 0
-        tokens = editor.tokenizedLineForScreenRow(1).tokens
-        scopeTree = editor.tokenizedLineForScreenRow(1).getScopeTree()
-        ensureValidScopeTree(scopeTree)