Merge branch 'master' into as-tiled-rendering

# Conflicts:
#	spec/text-editor-presenter-spec.coffee
#	src/lines-component.coffee
#	src/text-editor-presenter.coffee

src/lines-component.coffee

@@ -2,9 +2,12 @@ _ = require 'underscore-plus'
 {toArray} = require 'underscore-plus'
 {$$} = require 'space-pen'
 
+TokenIterator = require './token-iterator'
 DummyLineNode = $$(-> @div className: 'line', style: 'position: absolute; visibility: hidden;', => @span 'x')[0]
 AcceptFilter = {acceptNode: -> NodeFilter.FILTER_ACCEPT}
 WrapperDiv = document.createElement('div')
+TokenTextEscapeRegex = /[&"'<>]/g
+MaxTokenLength = 20000
 
 cloneObject = (object) ->
   clone = {}
@@ -16,6 +19,7 @@ class TileComponent
   placeholderTextDiv: null
 
   constructor: ({@presenter, @id}) ->
+    @tokenIterator = new TokenIterator
     @measuredLines = new Set
     @lineNodesByLineId = {}
     @screenRowsByLineId = {}
@@ -147,20 +151,116 @@ class TileComponent
       @buildEndOfLineHTML(id) or '&nbsp;'
 
   buildLineInnerHTML: (id) ->
-    {indentGuidesVisible} = @newState
-    {tokens, text, isOnlyWhitespace} = @newTileState.lines[id]
+    lineState = @newTileState.lines[id]
+    {firstNonWhitespaceIndex, firstTrailingWhitespaceIndex, invisibles} = lineState
+    lineIsWhitespaceOnly = firstTrailingWhitespaceIndex is 0
+
     innerHTML = ""
+    @tokenIterator.reset(lineState)
 
-    scopeStack = []
-    for token in tokens
-      innerHTML += @updateScopeStack(scopeStack, token.scopes)
-      hasIndentGuide = indentGuidesVisible and (token.hasLeadingWhitespace() or (token.hasTrailingWhitespace() and isOnlyWhitespace))
-      innerHTML += token.getValueAsHtml({hasIndentGuide})
+    while @tokenIterator.next()
+      for scope in @tokenIterator.getScopeEnds()
+        innerHTML += "</span>"
+
+      for scope in @tokenIterator.getScopeStarts()
+        innerHTML += "<span class=\"#{scope.replace(/\.+/g, ' ')}\">"
+
+      tokenStart = @tokenIterator.getScreenStart()
+      tokenEnd = @tokenIterator.getScreenEnd()
+      tokenText = @tokenIterator.getText()
+      isHardTab = @tokenIterator.isHardTab()
+
+      if hasLeadingWhitespace = tokenStart < firstNonWhitespaceIndex
+        tokenFirstNonWhitespaceIndex = firstNonWhitespaceIndex - tokenStart
+      else
+        tokenFirstNonWhitespaceIndex = null
+
+      if hasTrailingWhitespace = tokenEnd > firstTrailingWhitespaceIndex
+        tokenFirstTrailingWhitespaceIndex = Math.max(0, firstTrailingWhitespaceIndex - tokenStart)
+      else
+        tokenFirstTrailingWhitespaceIndex = null
+
+      hasIndentGuide =
+        @newState.indentGuidesVisible and
+          (hasLeadingWhitespace or lineIsWhitespaceOnly)
+
+      hasInvisibleCharacters =
+        (invisibles?.tab and isHardTab) or
+          (invisibles?.space and (hasLeadingWhitespace or hasTrailingWhitespace))
+
+      innerHTML += @buildTokenHTML(tokenText, isHardTab, tokenFirstNonWhitespaceIndex, tokenFirstTrailingWhitespaceIndex, hasIndentGuide, hasInvisibleCharacters)
+
+    for scope in @tokenIterator.getScopeEnds()
+      innerHTML += "</span>"
+
+    for scope in @tokenIterator.getScopes()
+      innerHTML += "</span>"
 
-    innerHTML += @popScope(scopeStack) while scopeStack.length > 0
     innerHTML += @buildEndOfLineHTML(id)
     innerHTML
 
+  buildTokenHTML: (tokenText, isHardTab, firstNonWhitespaceIndex, firstTrailingWhitespaceIndex, hasIndentGuide, hasInvisibleCharacters) ->
+    if isHardTab
+      classes = 'hard-tab'
+      classes += ' leading-whitespace' if firstNonWhitespaceIndex?
+      classes += ' trailing-whitespace' if firstTrailingWhitespaceIndex?
+      classes += ' indent-guide' if hasIndentGuide
+      classes += ' invisible-character' if hasInvisibleCharacters
+      return "<span class='#{classes}'>#{@escapeTokenText(tokenText)}</span>"
+    else
+      startIndex = 0
+      endIndex = tokenText.length
+
+      leadingHtml = ''
+      trailingHtml = ''
+
+      if firstNonWhitespaceIndex?
+        leadingWhitespace = tokenText.substring(0, firstNonWhitespaceIndex)
+
+        classes = 'leading-whitespace'
+        classes += ' indent-guide' if hasIndentGuide
+        classes += ' invisible-character' if hasInvisibleCharacters
+
+        leadingHtml = "<span class='#{classes}'>#{leadingWhitespace}</span>"
+        startIndex = firstNonWhitespaceIndex
+
+      if firstTrailingWhitespaceIndex?
+        tokenIsOnlyWhitespace = firstTrailingWhitespaceIndex is 0
+        trailingWhitespace = tokenText.substring(firstTrailingWhitespaceIndex)
+
+        classes = 'trailing-whitespace'
+        classes += ' indent-guide' if hasIndentGuide and not firstNonWhitespaceIndex? and tokenIsOnlyWhitespace
+        classes += ' invisible-character' if hasInvisibleCharacters
+
+        trailingHtml = "<span class='#{classes}'>#{trailingWhitespace}</span>"
+
+        endIndex = firstTrailingWhitespaceIndex
+
+      html = leadingHtml
+      if tokenText.length > MaxTokenLength
+        while startIndex < endIndex
+          html += "<span>" + @escapeTokenText(tokenText, startIndex, startIndex + MaxTokenLength) + "</span>"
+          startIndex += MaxTokenLength
+      else
+        html += @escapeTokenText(tokenText, startIndex, endIndex)
+
+      html += trailingHtml
+    html
+
+  escapeTokenText: (tokenText, startIndex, endIndex) ->
+    if startIndex? and endIndex? and startIndex > 0 or endIndex < tokenText.length
+      tokenText = tokenText.slice(startIndex, endIndex)
+    tokenText.replace(TokenTextEscapeRegex, @escapeTokenTextReplace)
+
+  escapeTokenTextReplace: (match) ->
+    switch match
+      when '&' then '&amp;'
+      when '"' then '&quot;'
+      when "'" then '&#39;'
+      when '<' then '&lt;'
+      when '>' then '&gt;'
+      else match
+
   buildEndOfLineHTML: (id) ->
     {endOfLineInvisibles} = @newTileState.lines[id]
 
@@ -170,31 +270,6 @@ class TileComponent
         html += "<span class='invisible-character'>#{invisible}</span>"
     html
 
-  updateScopeStack: (scopeStack, desiredScopeDescriptor) ->
-    html = ""
-
-    # Find a common prefix
-    for scope, i in desiredScopeDescriptor
-      break unless scopeStack[i] is desiredScopeDescriptor[i]
-
-    # Pop scopeDescriptor until we're at the common prefx
-    until scopeStack.length is i
-      html += @popScope(scopeStack)
-
-    # Push onto common prefix until scopeStack equals desiredScopeDescriptor
-    for j in [i...desiredScopeDescriptor.length]
-      html += @pushScope(scopeStack, desiredScopeDescriptor[j])
-
-    html
-
-  popScope: (scopeStack) ->
-    scopeStack.pop()
-    "</span>"
-
-  pushScope: (scopeStack, scope) ->
-    scopeStack.push(scope)
-    "<span class=\"#{scope.replace(/\.+/g, ' ')}\">"
-
   updateLineNode: (id) ->
     oldLineState = @oldTileState.lines[id]
     newLineState = @newTileState.lines[id]
@@ -243,19 +318,22 @@ class TileComponent
     iterator = null
     charIndex = 0
 
-    for {value, scopes, hasPairedCharacter} in tokenizedLine.tokens
+    @tokenIterator.reset(tokenizedLine)
+    while @tokenIterator.next()
+      scopes = @tokenIterator.getScopes()
+      text = @tokenIterator.getText()
       charWidths = @presenter.getScopedCharacterWidths(scopes)
 
-      valueIndex = 0
-      while valueIndex < value.length
-        if hasPairedCharacter
-          char = value.substr(valueIndex, 2)
+      textIndex = 0
+      while textIndex < text.length
+        if @tokenIterator.isPairedCharacter()
+          char = text
           charLength = 2
-          valueIndex += 2
+          textIndex += 2
         else
-          char = value[valueIndex]
+          char = text[textIndex]
           charLength = 1
-          valueIndex++
+          textIndex++
 
         continue if char is '\0'