Ignore mismatched tags between lines in TokenizedBufferIterator

Previously the DisplayLayer would have issues when closing tags that had
not been previously opened, requiring us to guard against mismatched
scopes at the frontier of asynchronous tokenization (see 5cfe9716 for
more information).

Now the DisplayLayer gracefully handles closing tags that had not been
opened, meaning we can eliminate this costly logic.

Signed-off-by: Nathan Sobo <nathan@github.com>

src/tokenized-buffer-iterator.js

@@ -6,7 +6,6 @@ module.exports = class TokenizedBufferIterator {
     this.tokenizedBuffer = tokenizedBuffer
     this.openScopeIds = null
     this.closeScopeIds = null
-    this.containingScopeIds = null
   }
 
   seek (position) {
@@ -16,9 +15,8 @@ module.exports = class TokenizedBufferIterator {
 
     const currentLine = this.tokenizedBuffer.tokenizedLineForRow(position.row)
     this.currentLineTags = currentLine.tags
-    this.currentLineOpenTags = currentLine.openScopes
     this.currentLineLength = currentLine.text.length
-    this.containingScopeIds = this.currentLineOpenTags.map((id) => fromFirstMateScopeId(id))
+    const containingScopeIds = currentLine.openScopes.map((id) => fromFirstMateScopeId(id))
 
     let currentColumn = 0
     for (let index = 0; index < this.currentLineTags.length; index++) {
@@ -31,11 +29,11 @@ module.exports = class TokenizedBufferIterator {
           currentColumn += tag
           while (this.closeScopeIds.length > 0) {
             this.closeScopeIds.shift()
-            this.containingScopeIds.pop()
+            containingScopeIds.pop()
           }
           while (this.openScopeIds.length > 0) {
             const openTag = this.openScopeIds.shift()
-            this.containingScopeIds.push(openTag)
+            containingScopeIds.push(openTag)
           }
         }
       } else {
@@ -48,11 +46,11 @@ module.exports = class TokenizedBufferIterator {
             } else {
               while (this.closeScopeIds.length > 0) {
                 this.closeScopeIds.shift()
-                this.containingScopeIds.pop()
+                containingScopeIds.pop()
               }
               while (this.openScopeIds.length > 0) {
                 const openTag = this.openScopeIds.shift()
-                this.containingScopeIds.push(openTag)
+                containingScopeIds.push(openTag)
               }
             }
           }
@@ -67,31 +65,16 @@ module.exports = class TokenizedBufferIterator {
       this.tagIndex = this.currentLineTags.length
     }
     this.position = Point(position.row, Math.min(this.currentLineLength, currentColumn))
-    return this.containingScopeIds.slice()
+    return containingScopeIds
   }
 
   moveToSuccessor () {
-    for (let i = 0; i < this.closeScopeIds.length; i++) {
-      this.containingScopeIds.pop()
-    }
-    for (let i = 0; i < this.openScopeIds.length; i++) {
-      const tag = this.openScopeIds[i]
-      this.containingScopeIds.push(tag)
-    }
     this.openScopeIds = []
     this.closeScopeIds = []
     while (true) {
       if (this.tagIndex === this.currentLineTags.length) {
         if (this.isAtTagBoundary()) {
           break
-        } else if (this.shouldMoveToNextLine) {
-          this.moveToNextLine()
-          this.openScopeIds = this.currentLineOpenTags.map((id) => fromFirstMateScopeId(id))
-          this.shouldMoveToNextLine = false
-        } else if (this.nextLineHasMismatchedContainingTags()) {
-          this.closeScopeIds = this.containingScopeIds.slice().reverse()
-          this.containingScopeIds = []
-          this.shouldMoveToNextLine = true
         } else if (!this.moveToNextLine()) {
           return false
         }
@@ -136,18 +119,6 @@ module.exports = class TokenizedBufferIterator {
     return this.openScopeIds.slice()
   }
 
-  nextLineHasMismatchedContainingTags () {
-    const line = this.tokenizedBuffer.tokenizedLineForRow(this.position.row + 1)
-    if (line == null) {
-      return false
-    } else {
-      return (
-        this.containingScopeIds.length !== line.openScopes.length ||
-        this.containingScopeIds.some((tag, i) => tag !== fromFirstMateScopeId(line.openScopes[i]))
-      )
-    }
-  }
-
   moveToNextLine () {
     this.position = Point(this.position.row + 1, 0)
     const tokenizedLine = this.tokenizedBuffer.tokenizedLineForRow(this.position.row)
@@ -156,7 +127,6 @@ module.exports = class TokenizedBufferIterator {
     } else {
       this.currentLineTags = tokenizedLine.tags
       this.currentLineLength = tokenizedLine.text.length
-      this.currentLineOpenTags = tokenizedLine.openScopes
       this.tagIndex = 0
       return true
     }