Merge pull request #11414 from atom/ns-switch-to-display-layers

Use display layers facility of text-buffer; delete all the code they replace

src/config-schema.coffee

@@ -155,6 +155,10 @@ module.exports =
         type: 'boolean'
         default: true
         description: 'Show line numbers in the editor\'s gutter.'
+      atomicSoftTabs:
+        type: 'boolean'
+        default: true
+        description: 'Skip over tab-length runs of leading whitespace when moving the cursor.'
       autoIndent:
         type: 'boolean'
         default: true

src/cursor.coffee

@@ -9,7 +9,7 @@ EmptyLineRegExp = /(\r\n[\t ]*\r\n)|(\n[\t ]*\n)/g
 # where text can be inserted.
 #
 # Cursors belong to {TextEditor}s and have some metadata attached in the form
-# of a {TextEditorMarker}.
+# of a {DisplayMarker}.
 module.exports =
 class Cursor extends Model
   screenPosition: null
@@ -129,7 +129,7 @@ class Cursor extends Model
   Section: Cursor Position Details
   ###
 
-  # Public: Returns the underlying {TextEditorMarker} for the cursor.
+  # Public: Returns the underlying {DisplayMarker} for the cursor.
   # Useful with overlay {Decoration}s.
   getMarker: -> @marker
 
@@ -261,11 +261,11 @@ class Cursor extends Model
 
       while columnCount > column and row > 0
         columnCount -= column
-        column = @editor.lineTextForScreenRow(--row).length
+        column = @editor.lineLengthForScreenRow(--row)
         columnCount-- # subtract 1 for the row move
 
       column = column - columnCount
-      @setScreenPosition({row, column}, clip: 'backward')
+      @setScreenPosition({row, column}, clipDirection: 'backward')
 
   # Public: Moves the cursor right one screen column.
   #
@@ -280,7 +280,7 @@ class Cursor extends Model
     else
       {row, column} = @getScreenPosition()
       maxLines = @editor.getScreenLineCount()
-      rowLength = @editor.lineTextForScreenRow(row).length
+      rowLength = @editor.lineLengthForScreenRow(row)
       columnsRemainingInLine = rowLength - column
 
       while columnCount > columnsRemainingInLine and row < maxLines - 1
@@ -288,11 +288,11 @@ class Cursor extends Model
         columnCount-- # subtract 1 for the row move
 
         column = 0
-        rowLength = @editor.lineTextForScreenRow(++row).length
+        rowLength = @editor.lineLengthForScreenRow(++row)
         columnsRemainingInLine = rowLength
 
       column = column + columnCount
-      @setScreenPosition({row, column}, clip: 'forward', wrapBeyondNewlines: true, wrapAtSoftNewlines: true)
+      @setScreenPosition({row, column}, clipDirection: 'forward')
 
   # Public: Moves the cursor to the top of the buffer.
   moveToTop: ->

src/decoration-manager.coffee

@@ -0,0 +1,181 @@
+{Emitter} = require 'event-kit'
+Model = require './model'
+Decoration = require './decoration'
+LayerDecoration = require './layer-decoration'
+
+module.exports =
+class DecorationManager extends Model
+  didUpdateDecorationsEventScheduled: false
+  updatedSynchronously: false
+
+  constructor: (@displayLayer, @defaultMarkerLayer) ->
+    super
+
+    @emitter = new Emitter
+    @decorationsById = {}
+    @decorationsByMarkerId = {}
+    @overlayDecorationsById = {}
+    @layerDecorationsByMarkerLayerId = {}
+    @decorationCountsByLayerId = {}
+    @layerUpdateDisposablesByLayerId = {}
+
+  observeDecorations: (callback) ->
+    callback(decoration) for decoration in @getDecorations()
+    @onDidAddDecoration(callback)
+
+  onDidAddDecoration: (callback) ->
+    @emitter.on 'did-add-decoration', callback
+
+  onDidRemoveDecoration: (callback) ->
+    @emitter.on 'did-remove-decoration', callback
+
+  onDidUpdateDecorations: (callback) ->
+    @emitter.on 'did-update-decorations', callback
+
+  setUpdatedSynchronously: (@updatedSynchronously) ->
+
+  decorationForId: (id) ->
+    @decorationsById[id]
+
+  getDecorations: (propertyFilter) ->
+    allDecorations = []
+    for markerId, decorations of @decorationsByMarkerId
+      allDecorations.push(decorations...) if decorations?
+    if propertyFilter?
+      allDecorations = allDecorations.filter (decoration) ->
+        for key, value of propertyFilter
+          return false unless decoration.properties[key] is value
+        true
+    allDecorations
+
+  getLineDecorations: (propertyFilter) ->
+    @getDecorations(propertyFilter).filter (decoration) -> decoration.isType('line')
+
+  getLineNumberDecorations: (propertyFilter) ->
+    @getDecorations(propertyFilter).filter (decoration) -> decoration.isType('line-number')
+
+  getHighlightDecorations: (propertyFilter) ->
+    @getDecorations(propertyFilter).filter (decoration) -> decoration.isType('highlight')
+
+  getOverlayDecorations: (propertyFilter) ->
+    result = []
+    for id, decoration of @overlayDecorationsById
+      result.push(decoration)
+    if propertyFilter?
+      result.filter (decoration) ->
+        for key, value of propertyFilter
+          return false unless decoration.properties[key] is value
+        true
+    else
+      result
+
+  decorationsForScreenRowRange: (startScreenRow, endScreenRow) ->
+    decorationsByMarkerId = {}
+    for marker in @defaultMarkerLayer.findMarkers(intersectsScreenRowRange: [startScreenRow, endScreenRow])
+      if decorations = @decorationsByMarkerId[marker.id]
+        decorationsByMarkerId[marker.id] = decorations
+    decorationsByMarkerId
+
+  decorationsStateForScreenRowRange: (startScreenRow, endScreenRow) ->
+    decorationsState = {}
+
+    for layerId of @decorationCountsByLayerId
+      layer = @displayLayer.getMarkerLayer(layerId)
+
+      for marker in layer.findMarkers(intersectsScreenRowRange: [startScreenRow, endScreenRow]) when marker.isValid()
+        screenRange = marker.getScreenRange()
+        bufferRange = marker.getBufferRange()
+        rangeIsReversed = marker.isReversed()
+
+        if decorations = @decorationsByMarkerId[marker.id]
+          for decoration in decorations
+            decorationsState[decoration.id] = {
+              properties: decoration.properties
+              screenRange, bufferRange, rangeIsReversed
+            }
+
+        if layerDecorations = @layerDecorationsByMarkerLayerId[layerId]
+          for layerDecoration in layerDecorations
+            decorationsState["#{layerDecoration.id}-#{marker.id}"] = {
+              properties: layerDecoration.overridePropertiesByMarkerId[marker.id] ? layerDecoration.properties
+              screenRange, bufferRange, rangeIsReversed
+            }
+
+    decorationsState
+
+  decorateMarker: (marker, decorationParams) ->
+    throw new Error("Cannot decorate a destroyed marker") if marker.isDestroyed()
+    marker = @displayLayer.getMarkerLayer(marker.layer.id).getMarker(marker.id)
+    decoration = new Decoration(marker, this, decorationParams)
+    @decorationsByMarkerId[marker.id] ?= []
+    @decorationsByMarkerId[marker.id].push(decoration)
+    @overlayDecorationsById[decoration.id] = decoration if decoration.isType('overlay')
+    @decorationsById[decoration.id] = decoration
+    @observeDecoratedLayer(marker.layer)
+    @scheduleUpdateDecorationsEvent()
+    @emitter.emit 'did-add-decoration', decoration
+    decoration
+
+  decorateMarkerLayer: (markerLayer, decorationParams) ->
+    decoration = new LayerDecoration(markerLayer, this, decorationParams)
+    @layerDecorationsByMarkerLayerId[markerLayer.id] ?= []
+    @layerDecorationsByMarkerLayerId[markerLayer.id].push(decoration)
+    @observeDecoratedLayer(markerLayer)
+    @scheduleUpdateDecorationsEvent()
+    decoration
+
+  decorationsForMarkerId: (markerId) ->
+    @decorationsByMarkerId[markerId]
+
+  scheduleUpdateDecorationsEvent: ->
+    if @updatedSynchronously
+      @emitter.emit 'did-update-decorations'
+      return
+
+    unless @didUpdateDecorationsEventScheduled
+      @didUpdateDecorationsEventScheduled = true
+      process.nextTick =>
+        @didUpdateDecorationsEventScheduled = false
+        @emitter.emit 'did-update-decorations'
+
+  decorationDidChangeType: (decoration) ->
+    if decoration.isType('overlay')
+      @overlayDecorationsById[decoration.id] = decoration
+    else
+      delete @overlayDecorationsById[decoration.id]
+
+  didDestroyDecoration: (decoration) ->
+    {marker} = decoration
+    return unless decorations = @decorationsByMarkerId[marker.id]
+    index = decorations.indexOf(decoration)
+
+    if index > -1
+      decorations.splice(index, 1)
+      delete @decorationsById[decoration.id]
+      @emitter.emit 'did-remove-decoration', decoration
+      delete @decorationsByMarkerId[marker.id] if decorations.length is 0
+      delete @overlayDecorationsById[decoration.id]
+      @unobserveDecoratedLayer(marker.layer)
+    @scheduleUpdateDecorationsEvent()
+
+  didDestroyLayerDecoration: (decoration) ->
+    {markerLayer} = decoration
+    return unless decorations = @layerDecorationsByMarkerLayerId[markerLayer.id]
+    index = decorations.indexOf(decoration)
+
+    if index > -1
+      decorations.splice(index, 1)
+      delete @layerDecorationsByMarkerLayerId[markerLayer.id] if decorations.length is 0
+      @unobserveDecoratedLayer(markerLayer)
+    @scheduleUpdateDecorationsEvent()
+
+  observeDecoratedLayer: (layer) ->
+    @decorationCountsByLayerId[layer.id] ?= 0
+    if ++@decorationCountsByLayerId[layer.id] is 1
+      @layerUpdateDisposablesByLayerId[layer.id] = layer.onDidUpdate(@scheduleUpdateDecorationsEvent.bind(this))
+
+  unobserveDecoratedLayer: (layer) ->
+    if --@decorationCountsByLayerId[layer.id] is 0
+      @layerUpdateDisposablesByLayerId[layer.id].dispose()
+      delete @decorationCountsByLayerId[layer.id]
+      delete @layerUpdateDisposablesByLayerId[layer.id]

src/decoration.coffee

@@ -11,7 +11,7 @@ translateDecorationParamsOldToNew = (decorationParams) ->
     decorationParams.gutterName = 'line-number'
   decorationParams
 
-# Essential: Represents a decoration that follows a {TextEditorMarker}. A decoration is
+# Essential: Represents a decoration that follows a {DisplayMarker}. A decoration is
 # basically a visual representation of a marker. It allows you to add CSS
 # classes to line numbers in the gutter, lines, and add selection-line regions
 # around marked ranges of text.
@@ -25,7 +25,7 @@ translateDecorationParamsOldToNew = (decorationParams) ->
 # decoration = editor.decorateMarker(marker, {type: 'line', class: 'my-line-class'})
 # ```
 #
-# Best practice for destroying the decoration is by destroying the {TextEditorMarker}.
+# Best practice for destroying the decoration is by destroying the {DisplayMarker}.
 #
 # ```coffee
 # marker.destroy()
@@ -62,7 +62,7 @@ class Decoration
   Section: Construction and Destruction
   ###
 
-  constructor: (@marker, @displayBuffer, properties) ->
+  constructor: (@marker, @decorationManager, properties) ->
     @emitter = new Emitter
     @id = nextId()
     @setProperties properties
@@ -71,14 +71,14 @@ class Decoration
 
   # Essential: Destroy this marker.
   #
-  # If you own the marker, you should use {TextEditorMarker::destroy} which will destroy
+  # If you own the marker, you should use {DisplayMarker::destroy} which will destroy
   # this decoration.
   destroy: ->
     return if @destroyed
     @markerDestroyDisposable.dispose()
     @markerDestroyDisposable = null
     @destroyed = true
-    @displayBuffer.didDestroyDecoration(this)
+    @decorationManager.didDestroyDecoration(this)
     @emitter.emit 'did-destroy'
     @emitter.dispose()
 
@@ -149,8 +149,8 @@ class Decoration
     oldProperties = @properties
     @properties = translateDecorationParamsOldToNew(newProperties)
     if newProperties.type?
-      @displayBuffer.decorationDidChangeType(this)
-    @displayBuffer.scheduleUpdateDecorationsEvent()
+      @decorationManager.decorationDidChangeType(this)
+    @decorationManager.scheduleUpdateDecorationsEvent()
     @emitter.emit 'did-change-properties', {oldProperties, newProperties}
 
   ###
@@ -175,5 +175,5 @@ class Decoration
     @properties.flashCount++
     @properties.flashClass = klass
     @properties.flashDuration = duration
-    @displayBuffer.scheduleUpdateDecorationsEvent()
+    @decorationManager.scheduleUpdateDecorationsEvent()
     @emitter.emit 'did-flash'

src/fold.coffee

@@ -1,83 +0,0 @@
-{Point, Range} = require 'text-buffer'
-
-# Represents a fold that collapses multiple buffer lines into a single
-# line on the screen.
-#
-# Their creation is managed by the {DisplayBuffer}.
-module.exports =
-class Fold
-  id: null
-  displayBuffer: null
-  marker: null
-
-  constructor: (@displayBuffer, @marker) ->
-    @id = @marker.id
-    @displayBuffer.foldsByMarkerId[@marker.id] = this
-    @marker.onDidDestroy => @destroyed()
-    @marker.onDidChange ({isValid}) => @destroy() unless isValid
-
-  # Returns whether this fold is contained within another fold
-  isInsideLargerFold: ->
-    largestContainingFoldMarker = @displayBuffer.findFoldMarker(containsRange: @getBufferRange())
-    largestContainingFoldMarker and
-      not largestContainingFoldMarker.getRange().isEqual(@getBufferRange())
-
-  # Destroys this fold
-  destroy: ->
-    @marker.destroy()
-
-  # Returns the fold's {Range} in buffer coordinates
-  #
-  # includeNewline - A {Boolean} which, if `true`, includes the trailing newline
-  #
-  # Returns a {Range}.
-  getBufferRange: ({includeNewline}={}) ->
-    range = @marker.getRange()
-
-    if range.end.row > range.start.row and nextFold = @displayBuffer.largestFoldStartingAtBufferRow(range.end.row)
-      nextRange = nextFold.getBufferRange()
-      range = new Range(range.start, nextRange.end)
-
-    if includeNewline
-      range = range.copy()
-      range.end.row++
-      range.end.column = 0
-    range
-
-  getBufferRowRange: ->
-    {start, end} = @getBufferRange()
-    [start.row, end.row]
-
-  # Returns the fold's start row as a {Number}.
-  getStartRow: ->
-    @getBufferRange().start.row
-
-  # Returns the fold's end row as a {Number}.
-  getEndRow: ->
-    @getBufferRange().end.row
-
-  # Returns a {String} representation of the fold.
-  inspect: ->
-    "Fold(#{@getStartRow()}, #{@getEndRow()})"
-
-  # Retrieves the number of buffer rows spanned by the fold.
-  #
-  # Returns a {Number}.
-  getBufferRowCount: ->
-    @getEndRow() - @getStartRow() + 1
-
-  # Identifies if a fold is nested within a fold.
-  #
-  # fold - A {Fold} to check
-  #
-  # Returns a {Boolean}.
-  isContainedByFold: (fold) ->
-    @isContainedByRange(fold.getBufferRange())
-
-  updateDisplayBuffer: ->
-    unless @isInsideLargerFold()
-      @displayBuffer.updateScreenLines(@getStartRow(), @getEndRow() + 1, 0, updateMarkers: true)
-
-  destroyed: ->
-    delete @displayBuffer.foldsByMarkerId[@marker.id]
-    @updateDisplayBuffer()

src/gutter.coffee

@@ -71,13 +71,13 @@ class Gutter
   isVisible: ->
     @visible
 
-  # Essential: Add a decoration that tracks a {TextEditorMarker}. When the marker moves,
+  # Essential: Add a decoration that tracks a {DisplayMarker}. When the marker moves,
   # is invalidated, or is destroyed, the decoration will be updated to reflect
   # the marker's state.
   #
   # ## Arguments
   #
-  # * `marker` A {TextEditorMarker} you want this decoration to follow.
+  # * `marker` A {DisplayMarker} you want this decoration to follow.
   # * `decorationParams` An {Object} representing the decoration. It is passed
   #   to {TextEditor::decorateMarker} as its `decorationParams` and so supports
   #   all options documented there.

src/language-mode.coffee

@@ -90,30 +90,36 @@ class LanguageMode
 
   # Folds all the foldable lines in the buffer.
   foldAll: ->
+    @unfoldAll()
+    foldedRowRanges = {}
     for currentRow in [0..@buffer.getLastRow()] by 1
-      [startRow, endRow] = @rowRangeForFoldAtBufferRow(currentRow) ? []
+      rowRange = [startRow, endRow] = @rowRangeForFoldAtBufferRow(currentRow) ? []
       continue unless startRow?
-      @editor.createFold(startRow, endRow)
+      continue if foldedRowRanges[rowRange]
+
+      @editor.foldBufferRowRange(startRow, endRow)
+      foldedRowRanges[rowRange] = true
     return
 
   # Unfolds all the foldable lines in the buffer.
   unfoldAll: ->
-    for fold in @editor.displayBuffer.foldsIntersectingBufferRowRange(0, @buffer.getLastRow()) by -1
-      fold.destroy()
-    return
+    @editor.displayLayer.destroyAllFolds()
 
   # Fold all comment and code blocks at a given indentLevel
   #
   # indentLevel - A {Number} indicating indentLevel; 0 based.
   foldAllAtIndentLevel: (indentLevel) ->
     @unfoldAll()
+    foldedRowRanges = {}
     for currentRow in [0..@buffer.getLastRow()] by 1
-      [startRow, endRow] = @rowRangeForFoldAtBufferRow(currentRow) ? []
+      rowRange = [startRow, endRow] = @rowRangeForFoldAtBufferRow(currentRow) ? []
       continue unless startRow?
+      continue if foldedRowRanges[rowRange]
 
       # assumption: startRow will always be the min indent level for the entire range
       if @editor.indentationForBufferRow(startRow) is indentLevel
-        @editor.createFold(startRow, endRow)
+        @editor.foldBufferRowRange(startRow, endRow)
+        foldedRowRanges[rowRange] = true
     return
 
   # Given a buffer row, creates a fold at it.
@@ -125,8 +131,8 @@ class LanguageMode
     for currentRow in [bufferRow..0] by -1
       [startRow, endRow] = @rowRangeForFoldAtBufferRow(currentRow) ? []
       continue unless startRow? and startRow <= bufferRow <= endRow
-      fold = @editor.displayBuffer.largestFoldStartingAtBufferRow(startRow)
-      return @editor.createFold(startRow, endRow) unless fold
+      unless @editor.isFoldedAtBufferRow(startRow)
+        return @editor.foldBufferRowRange(startRow, endRow)
 
   # Find the row range for a fold at a given bufferRow. Will handle comments
   # and code.
@@ -140,19 +146,19 @@ class LanguageMode
     rowRange
 
   rowRangeForCommentAtBufferRow: (bufferRow) ->
-    return unless @editor.displayBuffer.tokenizedBuffer.tokenizedLineForRow(bufferRow).isComment()
+    return unless @editor.tokenizedBuffer.tokenizedLineForRow(bufferRow).isComment()
 
     startRow = bufferRow
     endRow = bufferRow
 
     if bufferRow > 0
       for currentRow in [bufferRow-1..0] by -1
-        break unless @editor.displayBuffer.tokenizedBuffer.tokenizedLineForRow(currentRow).isComment()
+        break unless @editor.tokenizedBuffer.tokenizedLineForRow(currentRow).isComment()
         startRow = currentRow
 
     if bufferRow < @buffer.getLastRow()
       for currentRow in [bufferRow+1..@buffer.getLastRow()] by 1
-        break unless @editor.displayBuffer.tokenizedBuffer.tokenizedLineForRow(currentRow).isComment()
+        break unless @editor.tokenizedBuffer.tokenizedLineForRow(currentRow).isComment()
         endRow = currentRow
 
     return [startRow, endRow] if startRow isnt endRow
@@ -175,13 +181,13 @@ class LanguageMode
     [bufferRow, foldEndRow]
 
   isFoldableAtBufferRow: (bufferRow) ->
-    @editor.displayBuffer.tokenizedBuffer.isFoldableAtRow(bufferRow)
+    @editor.tokenizedBuffer.isFoldableAtRow(bufferRow)
 
   # Returns a {Boolean} indicating whether the line at the given buffer
   # row is a comment.
   isLineCommentedAtBufferRow: (bufferRow) ->
     return false unless 0 <= bufferRow <= @editor.getLastBufferRow()
-    @editor.displayBuffer.tokenizedBuffer.tokenizedLineForRow(bufferRow).isComment()
+    @editor.tokenizedBuffer.tokenizedLineForRow(bufferRow).isComment()
 
   # Find a row range for a 'paragraph' around specified bufferRow. A paragraph
   # is a block of text bounded by and empty line or a block of text that is not
@@ -234,11 +240,11 @@ class LanguageMode
   # Returns a {Number}.
   suggestedIndentForBufferRow: (bufferRow, options) ->
     line = @buffer.lineForRow(bufferRow)
-    tokenizedLine = @editor.displayBuffer.tokenizedBuffer.tokenizedLineForRow(bufferRow)
+    tokenizedLine = @editor.tokenizedBuffer.tokenizedLineForRow(bufferRow)
     @suggestedIndentForTokenizedLineAtBufferRow(bufferRow, line, tokenizedLine, options)
 
   suggestedIndentForLineAtBufferRow: (bufferRow, line, options) ->
-    tokenizedLine = @editor.displayBuffer.tokenizedBuffer.buildTokenizedLineForRowWithText(bufferRow, line)
+    tokenizedLine = @editor.tokenizedBuffer.buildTokenizedLineForRowWithText(bufferRow, line)
     @suggestedIndentForTokenizedLineAtBufferRow(bufferRow, line, tokenizedLine, options)
 
   suggestedIndentForTokenizedLineAtBufferRow: (bufferRow, line, tokenizedLine, options) ->

src/layer-decoration.coffee

@@ -7,7 +7,7 @@ nextId = -> idCounter++
 # layer. Created via {TextEditor::decorateMarkerLayer}.
 module.exports =
 class LayerDecoration
-  constructor: (@markerLayer, @displayBuffer, @properties) ->
+  constructor: (@markerLayer, @decorationManager, @properties) ->
     @id = nextId()
     @destroyed = false
     @markerLayerDestroyedDisposable = @markerLayer.onDidDestroy => @destroy()
@@ -19,7 +19,7 @@ class LayerDecoration
     @markerLayerDestroyedDisposable.dispose()
     @markerLayerDestroyedDisposable = null
     @destroyed = true
-    @displayBuffer.didDestroyLayerDecoration(this)
+    @decorationManager.didDestroyLayerDecoration(this)
 
   # Essential: Determine whether this decoration is destroyed.
   #
@@ -44,11 +44,11 @@ class LayerDecoration
   setProperties: (newProperties) ->
     return if @destroyed
     @properties = newProperties
-    @displayBuffer.scheduleUpdateDecorationsEvent()
+    @decorationManager.scheduleUpdateDecorationsEvent()
 
   # Essential: Override the decoration properties for a specific marker.
   #
-  # * `marker` The {TextEditorMarker} or {Marker} for which to override
+  # * `marker` The {DisplayMarker} or {Marker} for which to override
   #   properties.
   # * `properties` An {Object} containing properties to apply to this marker.
   #   Pass `null` to clear the override.
@@ -58,4 +58,4 @@ class LayerDecoration
       @overridePropertiesByMarkerId[marker.id] = properties
     else
       delete @overridePropertiesByMarkerId[marker.id]
-    @displayBuffer.scheduleUpdateDecorationsEvent()
+    @decorationManager.scheduleUpdateDecorationsEvent()

src/line-number-gutter-component.coffee

@@ -93,9 +93,9 @@ class LineNumberGutterComponent extends TiledComponent
     {target} = event
     lineNumber = target.parentNode
 
-    if target.classList.contains('icon-right') and lineNumber.classList.contains('foldable')
+    if target.classList.contains('icon-right')
       bufferRow = parseInt(lineNumber.getAttribute('data-buffer-row'))
       if lineNumber.classList.contains('folded')
         @editor.unfoldBufferRow(bufferRow)
-      else
+      else if lineNumber.classList.contains('foldable')
         @editor.foldBufferRow(bufferRow)

src/lines-component.coffee

@@ -43,7 +43,7 @@ class LinesComponent extends TiledComponent
     @domNode
 
   shouldRecreateAllTilesOnUpdate: ->
-    @oldState.indentGuidesVisible isnt @newState.indentGuidesVisible or @newState.continuousReflow
+    @newState.continuousReflow
 
   beforeUpdateSync: (state) ->
     if @newState.maxHeight isnt @oldState.maxHeight
@@ -70,8 +70,6 @@ class LinesComponent extends TiledComponent
 
     @cursorsComponent.updateSync(state)
 
-    @oldState.indentGuidesVisible = @newState.indentGuidesVisible
-
   buildComponentForTile: (id) -> new LinesTileComponent({id, @presenter, @domElementPool, @assert, @grammars})
 
   buildEmptyState: ->
@@ -97,10 +95,14 @@ class LinesComponent extends TiledComponent
     @presenter.setLineHeight(lineHeightInPixels)
     @presenter.setBaseCharacterWidth(defaultCharWidth, doubleWidthCharWidth, halfWidthCharWidth, koreanCharWidth)
 
-  lineNodeForLineIdAndScreenRow: (lineId, screenRow) ->
+  lineIdForScreenRow: (screenRow) ->
     tile = @presenter.tileForRow(screenRow)
-    @getComponentForTile(tile)?.lineNodeForLineId(lineId)
+    @getComponentForTile(tile)?.lineIdForScreenRow(screenRow)
 
-  textNodesForLineIdAndScreenRow: (lineId, screenRow) ->
+  lineNodeForScreenRow: (screenRow) ->
     tile = @presenter.tileForRow(screenRow)
-    @getComponentForTile(tile)?.textNodesForLineId(lineId)
+    @getComponentForTile(tile)?.lineNodeForScreenRow(screenRow)
+
+  textNodesForScreenRow: (screenRow) ->
+    tile = @presenter.tileForRow(screenRow)
+    @getComponentForTile(tile)?.textNodesForScreenRow(screenRow)

src/lines-tile-component.coffee

@@ -1,10 +1,10 @@
 _ = require 'underscore-plus'
 
 HighlightsComponent = require './highlights-component'
-TokenIterator = require './token-iterator'
 AcceptFilter = {acceptNode: -> NodeFilter.FILTER_ACCEPT}
 TokenTextEscapeRegex = /[&"'<>]/g
 MaxTokenLength = 20000
+ZERO_WIDTH_NBSP = '\ufeff'
 
 cloneObject = (object) ->
   clone = {}
@@ -14,7 +14,6 @@ cloneObject = (object) ->
 module.exports =
 class LinesTileComponent
   constructor: ({@presenter, @id, @domElementPool, @assert, grammars}) ->
-    @tokenIterator = new TokenIterator(grammarRegistry: grammars)
     @measuredLines = new Set
     @lineNodesByLineId = {}
     @screenRowsByLineId = {}
@@ -69,13 +68,10 @@ class LinesTileComponent
       @oldTileState.top = @newTileState.top
       @oldTileState.left = @newTileState.left
 
-    @removeLineNodes() unless @oldState.indentGuidesVisible is @newState.indentGuidesVisible
     @updateLineNodes()
 
     @highlightsComponent.updateSync(@newTileState)
 
-    @oldState.indentGuidesVisible = @newState.indentGuidesVisible
-
   removeLineNodes: ->
     @removeLineNode(id) for id of @oldTileState.lines
     return
@@ -195,8 +191,7 @@ class LinesTileComponent
   screenRowForNode: (node) -> parseInt(node.dataset.screenRow)
 
   buildLineNode: (id) ->
-    {width} = @newState
-    {screenRow, tokens, text, top, lineEnding, fold, isSoftWrapped, indentLevel, decorationClasses} = @newTileState.lines[id]
+    {lineText, tagCodes, screenRow, decorationClasses} = @newTileState.lines[id]
 
     lineNode = @domElementPool.buildElement("div", "line")
     lineNode.dataset.screenRow = screenRow
@@ -205,185 +200,40 @@ class LinesTileComponent
       for decorationClass in decorationClasses
         lineNode.classList.add(decorationClass)
 
-    @currentLineTextNodes = []
-    if text is ""
-      @setEmptyLineInnerNodes(id, lineNode)
-    else
-      @setLineInnerNodes(id, lineNode)
-    @textNodesByLineId[id] = @currentLineTextNodes
-
-    lineNode.appendChild(@domElementPool.buildElement("span", "fold-marker")) if fold
-    lineNode
-
-  setEmptyLineInnerNodes: (id, lineNode) ->
-    {indentGuidesVisible} = @newState
-    {indentLevel, tabLength, endOfLineInvisibles} = @newTileState.lines[id]
-
-    if indentGuidesVisible and indentLevel > 0
-      invisibleIndex = 0
-      for i in [0...indentLevel]
-        indentGuide = @domElementPool.buildElement("span", "indent-guide")
-        for j in [0...tabLength]
-          if invisible = endOfLineInvisibles?[invisibleIndex++]
-            invisibleSpan = @domElementPool.buildElement("span", "invisible-character")
-            textNode = @domElementPool.buildText(invisible)
-            invisibleSpan.appendChild(textNode)
-            indentGuide.appendChild(invisibleSpan)
-
-            @currentLineTextNodes.push(textNode)
-          else
-            textNode = @domElementPool.buildText(" ")
-            indentGuide.appendChild(textNode)
-
-            @currentLineTextNodes.push(textNode)
-        lineNode.appendChild(indentGuide)
-
-      while invisibleIndex < endOfLineInvisibles?.length
-        invisible = endOfLineInvisibles[invisibleIndex++]
-        invisibleSpan = @domElementPool.buildElement("span", "invisible-character")
-        textNode = @domElementPool.buildText(invisible)
-        invisibleSpan.appendChild(textNode)
-        lineNode.appendChild(invisibleSpan)
-
-        @currentLineTextNodes.push(textNode)
-    else
-      unless @appendEndOfLineNodes(id, lineNode)
-        textNode = @domElementPool.buildText("\u00a0")
-        lineNode.appendChild(textNode)
-
-        @currentLineTextNodes.push(textNode)
-
-  setLineInnerNodes: (id, lineNode) ->
-    lineState = @newTileState.lines[id]
-    {firstNonWhitespaceIndex, firstTrailingWhitespaceIndex, invisibles} = lineState
-    lineIsWhitespaceOnly = firstTrailingWhitespaceIndex is 0
-
-    @tokenIterator.reset(lineState)
+    textNodes = []
+    lineLength = 0
+    startIndex = 0
     openScopeNode = lineNode
-
-    while @tokenIterator.next()
-      for scope in @tokenIterator.getScopeEnds()
+    for tagCode in tagCodes when tagCode isnt 0
+      if @presenter.isCloseTagCode(tagCode)
         openScopeNode = openScopeNode.parentElement
-
-      for scope in @tokenIterator.getScopeStarts()
+      else if @presenter.isOpenTagCode(tagCode)
+        scope = @presenter.tagForCode(tagCode)
         newScopeNode = @domElementPool.buildElement("span", scope.replace(/\.+/g, ' '))
         openScopeNode.appendChild(newScopeNode)
         openScopeNode = newScopeNode
-
-      tokenStart = @tokenIterator.getScreenStart()
-      tokenEnd = @tokenIterator.getScreenEnd()
-      tokenText = @tokenIterator.getText()
-      isHardTab = @tokenIterator.isHardTab()
-
-      if hasLeadingWhitespace = tokenStart < firstNonWhitespaceIndex
-        tokenFirstNonWhitespaceIndex = firstNonWhitespaceIndex - tokenStart
       else
-        tokenFirstNonWhitespaceIndex = null
+        textNode = @domElementPool.buildText(lineText.substr(startIndex, tagCode))
+        startIndex += tagCode
+        openScopeNode.appendChild(textNode)
+        textNodes.push(textNode)
 
-      if hasTrailingWhitespace = tokenEnd > firstTrailingWhitespaceIndex
-        tokenFirstTrailingWhitespaceIndex = Math.max(0, firstTrailingWhitespaceIndex - tokenStart)
-      else
-        tokenFirstTrailingWhitespaceIndex = null
+    if startIndex is 0
+      textNode = @domElementPool.buildText(' ')
+      lineNode.appendChild(textNode)
+      textNodes.push(textNode)
 
-      hasIndentGuide =
-        @newState.indentGuidesVisible and
-          (hasLeadingWhitespace or lineIsWhitespaceOnly)
+    if lineText.endsWith(@presenter.displayLayer.foldCharacter)
+      # Insert a zero-width non-breaking whitespace, so that
+      # LinesYardstick can take the fold-marker::after pseudo-element
+      # into account during measurements when such marker is the last
+      # character on the line.
+      textNode = @domElementPool.buildText(ZERO_WIDTH_NBSP)
+      lineNode.appendChild(textNode)
+      textNodes.push(textNode)
 
-      hasInvisibleCharacters =
-        (invisibles?.tab and isHardTab) or
-          (invisibles?.space and (hasLeadingWhitespace or hasTrailingWhitespace))
-
-      @appendTokenNodes(tokenText, isHardTab, tokenFirstNonWhitespaceIndex, tokenFirstTrailingWhitespaceIndex, hasIndentGuide, hasInvisibleCharacters, openScopeNode)
-
-    @appendEndOfLineNodes(id, lineNode)
-
-  appendTokenNodes: (tokenText, isHardTab, firstNonWhitespaceIndex, firstTrailingWhitespaceIndex, hasIndentGuide, hasInvisibleCharacters, scopeNode) ->
-    if isHardTab
-      textNode = @domElementPool.buildText(tokenText)
-      hardTabNode = @domElementPool.buildElement("span", "hard-tab")
-      hardTabNode.classList.add("leading-whitespace") if firstNonWhitespaceIndex?
-      hardTabNode.classList.add("trailing-whitespace") if firstTrailingWhitespaceIndex?
-      hardTabNode.classList.add("indent-guide") if hasIndentGuide
-      hardTabNode.classList.add("invisible-character") if hasInvisibleCharacters
-      hardTabNode.appendChild(textNode)
-
-      scopeNode.appendChild(hardTabNode)
-      @currentLineTextNodes.push(textNode)
-    else
-      startIndex = 0
-      endIndex = tokenText.length
-
-      leadingWhitespaceNode = null
-      leadingWhitespaceTextNode = null
-      trailingWhitespaceNode = null
-      trailingWhitespaceTextNode = null
-
-      if firstNonWhitespaceIndex?
-        leadingWhitespaceTextNode =
-          @domElementPool.buildText(tokenText.substring(0, firstNonWhitespaceIndex))
-        leadingWhitespaceNode = @domElementPool.buildElement("span", "leading-whitespace")
-        leadingWhitespaceNode.classList.add("indent-guide") if hasIndentGuide
-        leadingWhitespaceNode.classList.add("invisible-character") if hasInvisibleCharacters
-        leadingWhitespaceNode.appendChild(leadingWhitespaceTextNode)
-
-        startIndex = firstNonWhitespaceIndex
-
-      if firstTrailingWhitespaceIndex?
-        tokenIsOnlyWhitespace = firstTrailingWhitespaceIndex is 0
-
-        trailingWhitespaceTextNode =
-          @domElementPool.buildText(tokenText.substring(firstTrailingWhitespaceIndex))
-        trailingWhitespaceNode = @domElementPool.buildElement("span", "trailing-whitespace")
-        trailingWhitespaceNode.classList.add("indent-guide") if hasIndentGuide and not firstNonWhitespaceIndex? and tokenIsOnlyWhitespace
-        trailingWhitespaceNode.classList.add("invisible-character") if hasInvisibleCharacters
-        trailingWhitespaceNode.appendChild(trailingWhitespaceTextNode)
-
-        endIndex = firstTrailingWhitespaceIndex
-
-      if leadingWhitespaceNode?
-        scopeNode.appendChild(leadingWhitespaceNode)
-        @currentLineTextNodes.push(leadingWhitespaceTextNode)
-
-      if tokenText.length > MaxTokenLength
-        while startIndex < endIndex
-          textNode = @domElementPool.buildText(
-            @sliceText(tokenText, startIndex, startIndex + MaxTokenLength)
-          )
-          textSpan = @domElementPool.buildElement("span")
-
-          textSpan.appendChild(textNode)
-          scopeNode.appendChild(textSpan)
-          startIndex += MaxTokenLength
-          @currentLineTextNodes.push(textNode)
-      else
-        textNode = @domElementPool.buildText(@sliceText(tokenText, startIndex, endIndex))
-        scopeNode.appendChild(textNode)
-        @currentLineTextNodes.push(textNode)
-
-      if trailingWhitespaceNode?
-        scopeNode.appendChild(trailingWhitespaceNode)
-        @currentLineTextNodes.push(trailingWhitespaceTextNode)
-
-  sliceText: (tokenText, startIndex, endIndex) ->
-    if startIndex? and endIndex? and startIndex > 0 or endIndex < tokenText.length
-      tokenText = tokenText.slice(startIndex, endIndex)
-    tokenText
-
-  appendEndOfLineNodes: (id, lineNode) ->
-    {endOfLineInvisibles} = @newTileState.lines[id]
-
-    hasInvisibles = false
-    if endOfLineInvisibles?
-      for invisible in endOfLineInvisibles
-        hasInvisibles = true
-        invisibleSpan = @domElementPool.buildElement("span", "invisible-character")
-        textNode = @domElementPool.buildText(invisible)
-        invisibleSpan.appendChild(textNode)
-        lineNode.appendChild(invisibleSpan)
-
-        @currentLineTextNodes.push(textNode)
-
-    hasInvisibles
+    @textNodesByLineId[id] = textNodes
+    lineNode
 
   updateLineNode: (id) ->
     oldLineState = @oldTileState.lines[id]
@@ -436,3 +286,9 @@ class LinesTileComponent
 
   textNodesForLineId: (lineId) ->
     @textNodesByLineId[lineId].slice()
+
+  lineIdForScreenRow: (screenRow) ->
+    @lineIdsByScreenRow[screenRow]
+
+  textNodesForScreenRow: (screenRow) ->
+    @textNodesByLineId[@lineIdsByScreenRow[screenRow]]?.slice()

src/lines-yardstick.coffee

@@ -1,15 +1,14 @@
-TokenIterator = require './token-iterator'
 {Point} = require 'text-buffer'
+{isPairedCharacter} = require './text-utils'
 
 module.exports =
 class LinesYardstick
   constructor: (@model, @lineNodesProvider, @lineTopIndex, grammarRegistry) ->
-    @tokenIterator = new TokenIterator({grammarRegistry})
     @rangeForMeasurement = document.createRange()
     @invalidateCache()
 
   invalidateCache: ->
-    @pixelPositionsByLineIdAndColumn = {}
+    @leftPixelPositionCache = {}
 
   measuredRowForPixelPosition: (pixelPosition) ->
     targetTop = pixelPosition.top
@@ -21,61 +20,63 @@ class LinesYardstick
     targetLeft = pixelPosition.left
     defaultCharWidth = @model.getDefaultCharWidth()
     row = @lineTopIndex.rowForPixelPosition(targetTop)
-    targetLeft = 0 if targetTop < 0
+    targetLeft = 0 if targetTop < 0 or targetLeft < 0
     targetLeft = Infinity if row > @model.getLastScreenRow()
     row = Math.min(row, @model.getLastScreenRow())
     row = Math.max(0, row)
 
-    line = @model.tokenizedLineForScreenRow(row)
-    lineNode = @lineNodesProvider.lineNodeForLineIdAndScreenRow(line?.id, row)
+    lineNode = @lineNodesProvider.lineNodeForScreenRow(row)
+    return Point(row, 0) unless lineNode
 
-    return Point(row, 0) unless lineNode? and line?
+    textNodes = @lineNodesProvider.textNodesForScreenRow(row)
+    lineOffset = lineNode.getBoundingClientRect().left
+    targetLeft += lineOffset
 
-    textNodes = @lineNodesProvider.textNodesForLineIdAndScreenRow(line.id, row)
-    column = 0
-    previousColumn = 0
-    previousLeft = 0
+    textNodeIndex = 0
+    low = 0
+    high = textNodes.length - 1
+    while low <= high
+      mid = low + (high - low >> 1)
+      textNode = textNodes[mid]
+      rangeRect = @clientRectForRange(textNode, 0, textNode.length)
+      if targetLeft < rangeRect.left
+        high = mid - 1
+        textNodeIndex = Math.max(0, mid - 1)
+      else if targetLeft > rangeRect.right
+        low = mid + 1
+        textNodeIndex = Math.min(textNodes.length - 1, mid + 1)
+      else
+        textNodeIndex = mid
+        break
 
-    @tokenIterator.reset(line, false)
-    while @tokenIterator.next()
-      text = @tokenIterator.getText()
-      textIndex = 0
-      while textIndex < text.length
-        if @tokenIterator.isPairedCharacter()
-          char = text
-          charLength = 2
-          textIndex += 2
+    textNode = textNodes[textNodeIndex]
+    characterIndex = 0
+    low = 0
+    high = textNode.textContent.length - 1
+    while low <= high
+      charIndex = low + (high - low >> 1)
+      if isPairedCharacter(textNode.textContent, charIndex)
+        nextCharIndex = charIndex + 2
+      else
+        nextCharIndex = charIndex + 1
+
+      rangeRect = @clientRectForRange(textNode, charIndex, nextCharIndex)
+      if targetLeft < rangeRect.left
+        high = charIndex - 1
+        characterIndex = Math.max(0, charIndex - 1)
+      else if targetLeft > rangeRect.right
+        low = nextCharIndex
+        characterIndex = Math.min(textNode.textContent.length, nextCharIndex)
+      else
+        if targetLeft <= ((rangeRect.left + rangeRect.right) / 2)
+          characterIndex = charIndex
         else
-          char = text[textIndex]
-          charLength = 1
-          textIndex++
+          characterIndex = nextCharIndex
+        break
 
-        unless textNode?
-          textNode = textNodes.shift()
-          textNodeLength = textNode.textContent.length
-          textNodeIndex = 0
-          nextTextNodeIndex = textNodeLength
-
-        while nextTextNodeIndex <= column
-          textNode = textNodes.shift()
-          textNodeLength = textNode.textContent.length
-          textNodeIndex = nextTextNodeIndex
-          nextTextNodeIndex = textNodeIndex + textNodeLength
-
-        indexWithinTextNode = column - textNodeIndex
-        left = @leftPixelPositionForCharInTextNode(lineNode, textNode, indexWithinTextNode)
-        charWidth = left - previousLeft
-
-        return Point(row, previousColumn) if targetLeft <= previousLeft + (charWidth / 2)
-
-        previousLeft = left
-        previousColumn = column
-        column += charLength
-
-    if targetLeft <= previousLeft + (charWidth / 2)
-      Point(row, previousColumn)
-    else
-      Point(row, column)
+    textNodeStartColumn = 0
+    textNodeStartColumn += textNodes[i].length for i in [0...textNodeIndex] by 1
+    Point(row, textNodeStartColumn + characterIndex)
 
   pixelPositionForScreenPosition: (screenPosition) ->
     targetRow = screenPosition.row
@@ -87,76 +88,41 @@ class LinesYardstick
     {top, left}
 
   leftPixelPositionForScreenPosition: (row, column) ->
-    line = @model.tokenizedLineForScreenRow(row)
-    lineNode = @lineNodesProvider.lineNodeForLineIdAndScreenRow(line?.id, row)
+    lineNode = @lineNodesProvider.lineNodeForScreenRow(row)
+    lineId = @lineNodesProvider.lineIdForScreenRow(row)
 
-    return 0 unless line? and lineNode?
+    return 0 unless lineNode?
 
-    if cachedPosition = @pixelPositionsByLineIdAndColumn[line.id]?[column]
+    if cachedPosition = @leftPixelPositionCache[lineId]?[column]
       return cachedPosition
 
-    textNodes = @lineNodesProvider.textNodesForLineIdAndScreenRow(line.id, row)
-    indexWithinTextNode = null
-    charIndex = 0
+    textNodes = @lineNodesProvider.textNodesForScreenRow(row)
+    textNodeStartColumn = 0
 
-    @tokenIterator.reset(line, false)
-    while @tokenIterator.next()
-      break if foundIndexWithinTextNode?
-
-      text = @tokenIterator.getText()
-
-      textIndex = 0
-      while textIndex < text.length
-        if @tokenIterator.isPairedCharacter()
-          char = text
-          charLength = 2
-          textIndex += 2
-        else
-          char = text[textIndex]
-          charLength = 1
-          textIndex++
-
-        unless textNode?
-          textNode = textNodes.shift()
-          textNodeLength = textNode.textContent.length
-          textNodeIndex = 0
-          nextTextNodeIndex = textNodeLength
-
-        while nextTextNodeIndex <= charIndex
-          textNode = textNodes.shift()
-          textNodeLength = textNode.textContent.length
-          textNodeIndex = nextTextNodeIndex
-          nextTextNodeIndex = textNodeIndex + textNodeLength
-
-        if charIndex is column
-          foundIndexWithinTextNode = charIndex - textNodeIndex
-          break
-
-        charIndex += charLength
+    for textNode in textNodes
+      textNodeEndColumn = textNodeStartColumn + textNode.textContent.length
+      if textNodeEndColumn > column
+        indexInTextNode = column - textNodeStartColumn
+        break
+      else
+        textNodeStartColumn = textNodeEndColumn
 
     if textNode?
-      foundIndexWithinTextNode ?= textNode.textContent.length
-      position = @leftPixelPositionForCharInTextNode(
-        lineNode, textNode, foundIndexWithinTextNode
-      )
-      @pixelPositionsByLineIdAndColumn[line.id] ?= {}
-      @pixelPositionsByLineIdAndColumn[line.id][column] = position
-      position
+      indexInTextNode ?= textNode.textContent.length
+      lineOffset = lineNode.getBoundingClientRect().left
+      if indexInTextNode is 0
+        leftPixelPosition = @clientRectForRange(textNode, 0, 1).left
+      else
+        leftPixelPosition = @clientRectForRange(textNode, 0, indexInTextNode).right
+      leftPixelPosition -= lineOffset
+
+      @leftPixelPositionCache[lineId] ?= {}
+      @leftPixelPositionCache[lineId][column] = leftPixelPosition
+      leftPixelPosition
     else
       0
 
-  leftPixelPositionForCharInTextNode: (lineNode, textNode, charIndex) ->
-    if charIndex is 0
-      width = 0
-    else
-      @rangeForMeasurement.setStart(textNode, 0)
-      @rangeForMeasurement.setEnd(textNode, charIndex)
-      width = @rangeForMeasurement.getBoundingClientRect().width
-
-    @rangeForMeasurement.setStart(textNode, 0)
-    @rangeForMeasurement.setEnd(textNode, textNode.textContent.length)
-    left = @rangeForMeasurement.getBoundingClientRect().left
-
-    offset = lineNode.getBoundingClientRect().left
-
-    left + width - offset
+  clientRectForRange: (textNode, startIndex, endIndex) ->
+    @rangeForMeasurement.setStart(textNode, startIndex)
+    @rangeForMeasurement.setEnd(textNode, endIndex)
+    @rangeForMeasurement.getClientRects()[0] ? @rangeForMeasurement.getBoundingClientRect()

src/marker-observation-window.coffee

@@ -1,9 +1,9 @@
 module.exports =
 class MarkerObservationWindow
-  constructor: (@displayBuffer, @bufferWindow) ->
+  constructor: (@decorationManager, @bufferWindow) ->
 
   setScreenRange: (range) ->
-    @bufferWindow.setRange(@displayBuffer.bufferRangeForScreenRange(range))
+    @bufferWindow.setRange(@decorationManager.bufferRangeForScreenRange(range))
 
   setBufferRange: (range) ->
     @bufferWindow.setRange(range)

src/selection.coffee

@@ -87,7 +87,7 @@ class Selection extends Model
   setBufferRange: (bufferRange, options={}) ->
     bufferRange = Range.fromObject(bufferRange)
     options.reversed ?= @isReversed()
-    @editor.destroyFoldsContainingBufferRange(bufferRange) unless options.preserveFolds
+    @editor.destroyFoldsIntersectingBufferRange(bufferRange) unless options.preserveFolds
     @modifySelection =>
       needsFlash = options.flash
       delete options.flash if options.flash?
@@ -174,7 +174,7 @@ class Selection extends Model
   #     range. Defaults to `true` if this is the most recently added selection,
   #     `false` otherwise.
   clear: (options) ->
-    @marker.setProperties(goalScreenRange: null)
+    @goalScreenRange = null
     @marker.clearTail() unless @retainSelection
     @autoscroll() if options?.autoscroll ? @isLastSelection()
     @finalize()
@@ -365,7 +365,6 @@ class Selection extends Model
   #   * `undo` if `skip`, skips the undo stack for this operation.
   insertText: (text, options={}) ->
     oldBufferRange = @getBufferRange()
-    @editor.unfoldBufferRow(oldBufferRange.end.row)
     wasReversed = @isReversed()
     @clear()
 
@@ -394,7 +393,7 @@ class Selection extends Model
     if options.select
       @setBufferRange(newBufferRange, reversed: wasReversed)
     else
-      @cursor.setBufferPosition(newBufferRange.end, clip: 'forward') if wasReversed
+      @cursor.setBufferPosition(newBufferRange.end) if wasReversed
 
     if autoIndentFirstLine
       @editor.setIndentationForBufferRow(oldBufferRange.start.row, desiredIndentLevel)
@@ -411,7 +410,7 @@ class Selection extends Model
   # Public: Removes the first character before the selection if the selection
   # is empty otherwise it deletes the selection.
   backspace: ->
-    @selectLeft() if @isEmpty() and not @editor.isFoldedAtScreenRow(@cursor.getScreenRow())
+    @selectLeft() if @isEmpty()
     @deleteSelectedText()
 
   # Public: Removes the selection or, if nothing is selected, then all
@@ -446,11 +445,7 @@ class Selection extends Model
   # Public: Removes the selection or the next character after the start of the
   # selection if the selection is empty.
   delete: ->
-    if @isEmpty()
-      if @cursor.isAtEndOfLine() and fold = @editor.largestFoldStartingAtScreenRow(@cursor.getScreenRow() + 1)
-        @selectToBufferPosition(fold.getBufferRange().end)
-      else
-        @selectRight()
+    @selectRight() if @isEmpty()
     @deleteSelectedText()
 
   # Public: If the selection is empty, removes all text from the cursor to the
@@ -483,8 +478,6 @@ class Selection extends Model
   # Public: Removes only the selected text.
   deleteSelectedText: ->
     bufferRange = @getBufferRange()
-    if bufferRange.isEmpty() and fold = @editor.largestFoldContainingBufferRow(bufferRange.start.row)
-      bufferRange = bufferRange.union(fold.getBufferRange(includeNewline: true))
     @editor.buffer.delete(bufferRange) unless bufferRange.isEmpty()
     @cursor?.setBufferPosition(bufferRange.start)
 
@@ -516,7 +509,7 @@ class Selection extends Model
     if selectedRange.isEmpty()
       return if selectedRange.start.row is @editor.buffer.getLastRow()
     else
-      joinMarker = @editor.markBufferRange(selectedRange, invalidationStrategy: 'never')
+      joinMarker = @editor.markBufferRange(selectedRange, invalidate: 'never')
 
     rowCount = Math.max(1, selectedRange.getRowCount() - 1)
     for row in [0...rowCount]
@@ -635,8 +628,9 @@ class Selection extends Model
   # Public: Creates a fold containing the current selection.
   fold: ->
     range = @getBufferRange()
-    @editor.createFold(range.start.row, range.end.row)
-    @cursor.setBufferPosition([range.end.row + 1, 0])
+    unless range.isEmpty()
+      @editor.foldBufferRange(range)
+      @cursor.setBufferPosition(range.end)
 
   # Private: Increase the indentation level of the given text by given number
   # of levels. Leaves the first line unchanged.
@@ -690,7 +684,7 @@ class Selection extends Model
 
   # Public: Moves the selection down one row.
   addSelectionBelow: ->
-    range = (@getGoalScreenRange() ? @getScreenRange()).copy()
+    range = @getGoalScreenRange().copy()
     nextRow = range.end.row + 1
 
     for row in [nextRow..@editor.getLastScreenRow()]
@@ -703,14 +697,15 @@ class Selection extends Model
       else
         continue if clippedRange.isEmpty()
 
-      @editor.addSelectionForScreenRange(clippedRange, goalScreenRange: range)
+      selection = @editor.addSelectionForScreenRange(clippedRange)
+      selection.setGoalScreenRange(range)
       break
 
     return
 
   # Public: Moves the selection up one row.
   addSelectionAbove: ->
-    range = (@getGoalScreenRange() ? @getScreenRange()).copy()
+    range = @getGoalScreenRange().copy()
     previousRow = range.end.row - 1
 
     for row in [previousRow..0]
@@ -723,7 +718,8 @@ class Selection extends Model
       else
         continue if clippedRange.isEmpty()
 
-      @editor.addSelectionForScreenRange(clippedRange, goalScreenRange: range)
+      selection = @editor.addSelectionForScreenRange(clippedRange)
+      selection.setGoalScreenRange(range)
       break
 
     return
@@ -762,6 +758,12 @@ class Selection extends Model
   Section: Private Utilities
   ###
 
+  setGoalScreenRange: (range) ->
+    @goalScreenRange = Range.fromObject(range)
+
+  getGoalScreenRange: ->
+    @goalScreenRange ? @getScreenRange()
+
   markerDidChange: (e) ->
     {oldHeadBufferPosition, oldTailBufferPosition, newHeadBufferPosition} = e
     {oldHeadScreenPosition, oldTailScreenPosition, newHeadScreenPosition} = e
@@ -832,7 +834,3 @@ class Selection extends Model
   # Returns a {Point} representing the new tail position.
   plantTail: ->
     @marker.plantTail()
-
-  getGoalScreenRange: ->
-    if goalScreenRange = @marker.getProperties().goalScreenRange
-      Range.fromObject(goalScreenRange)

src/text-editor-component.coffee

@@ -494,7 +494,7 @@ class TextEditorComponent
     unless @presenter.isRowVisible(screenPosition.row)
       @presenter.setScreenRowsToMeasure([screenPosition.row])
 
-    unless @linesComponent.lineNodeForLineIdAndScreenRow(@presenter.lineIdForScreenRow(screenPosition.row), screenPosition.row)?
+    unless @linesComponent.lineNodeForScreenRow(screenPosition.row)?
       @updateSyncPreMeasurement()
 
     pixelPosition = @linesYardstick.pixelPositionForScreenPosition(screenPosition)
@@ -560,8 +560,8 @@ class TextEditorComponent
     screenPosition = @screenPositionForMouseEvent(event)
 
     if event.target?.classList.contains('fold-marker')
-      bufferRow = @editor.bufferRowForScreenRow(screenPosition.row)
-      @editor.unfoldBufferRow(bufferRow)
+      bufferPosition = @editor.bufferPositionForScreenPosition(screenPosition)
+      @editor.destroyFoldsIntersectingBufferRange([bufferPosition, bufferPosition])
       return
 
     switch detail
@@ -607,7 +607,7 @@ class TextEditorComponent
     clickedScreenRow = @screenPositionForMouseEvent(event).row
     clickedBufferRow = @editor.bufferRowForScreenRow(clickedScreenRow)
     initialScreenRange = @editor.screenRangeForBufferRange([[clickedBufferRow, 0], [clickedBufferRow + 1, 0]])
-    @editor.addSelectionForScreenRange(initialScreenRange, preserveFolds: true, autoscroll: false)
+    @editor.addSelectionForScreenRange(initialScreenRange, autoscroll: false)
     @handleGutterDrag(initialScreenRange)
 
   onGutterShiftClick: (event) =>
@@ -890,10 +890,7 @@ class TextEditorComponent
     e.abortKeyBinding() unless @editor.consolidateSelections()
 
   lineNodeForScreenRow: (screenRow) ->
-    tileRow = @presenter.tileForRow(screenRow)
-    tileComponent = @linesComponent.getComponentForTile(tileRow)
-
-    tileComponent?.lineNodeForScreenRow(screenRow)
+    @linesComponent.lineNodeForScreenRow(screenRow)
 
   lineNumberNodeForScreenRow: (screenRow) ->
     tileRow = @presenter.tileForRow(screenRow)
@@ -950,7 +947,7 @@ class TextEditorComponent
 
   screenPositionForMouseEvent: (event, linesClientRect) ->
     pixelPosition = @pixelPositionForMouseEvent(event, linesClientRect)
-    @screenPositionForPixelPosition(pixelPosition, true)
+    @screenPositionForPixelPosition(pixelPosition)
 
   pixelPositionForMouseEvent: (event, linesClientRect) ->
     {clientX, clientY} = event

src/text-editor-marker-layer.coffee

@@ -1,192 +0,0 @@
-TextEditorMarker = require './text-editor-marker'
-
-# Public: *Experimental:* A container for a related set of markers at the
-# {TextEditor} level. Wraps an underlying {MarkerLayer} on the editor's
-# {TextBuffer}.
-#
-# This API is experimental and subject to change on any release.
-module.exports =
-class TextEditorMarkerLayer
-  constructor: (@displayBuffer, @bufferMarkerLayer, @isDefaultLayer) ->
-    @id = @bufferMarkerLayer.id
-    @markersById = {}
-
-  ###
-  Section: Lifecycle
-  ###
-
-  # Essential: Destroy this layer.
-  destroy: ->
-    if @isDefaultLayer
-      marker.destroy() for id, marker of @markersById
-    else
-      @bufferMarkerLayer.destroy()
-
-  ###
-  Section: Querying
-  ###
-
-  # Essential: Get an existing marker by its id.
-  #
-  # Returns a {TextEditorMarker}.
-  getMarker: (id) ->
-    if editorMarker = @markersById[id]
-      editorMarker
-    else if bufferMarker = @bufferMarkerLayer.getMarker(id)
-      @markersById[id] = new TextEditorMarker(this, bufferMarker)
-
-  # Essential: Get all markers in the layer.
-  #
-  # Returns an {Array} of {TextEditorMarker}s.
-  getMarkers: ->
-    @bufferMarkerLayer.getMarkers().map ({id}) => @getMarker(id)
-
-  # Public: Get the number of markers in the marker layer.
-  #
-  # Returns a {Number}.
-  getMarkerCount: ->
-    @bufferMarkerLayer.getMarkerCount()
-
-  # Public: Find markers in the layer conforming to the given parameters.
-  #
-  # See the documentation for {TextEditor::findMarkers}.
-  findMarkers: (params) ->
-    params = @translateToBufferMarkerParams(params)
-    @bufferMarkerLayer.findMarkers(params).map (stringMarker) => @getMarker(stringMarker.id)
-
-  ###
-  Section: Marker creation
-  ###
-
-  # Essential: Create a marker on this layer with the given range in buffer
-  # coordinates.
-  #
-  # See the documentation for {TextEditor::markBufferRange}
-  markBufferRange: (bufferRange, options) ->
-    @getMarker(@bufferMarkerLayer.markRange(bufferRange, options).id)
-
-  # Essential: Create a marker on this layer with the given range in screen
-  # coordinates.
-  #
-  # See the documentation for {TextEditor::markScreenRange}
-  markScreenRange: (screenRange, options) ->
-    bufferRange = @displayBuffer.bufferRangeForScreenRange(screenRange)
-    @markBufferRange(bufferRange, options)
-
-  # Public: Create a marker on this layer with the given buffer position and no
-  # tail.
-  #
-  # See the documentation for {TextEditor::markBufferPosition}
-  markBufferPosition: (bufferPosition, options) ->
-    @getMarker(@bufferMarkerLayer.markPosition(bufferPosition, options).id)
-
-  # Public: Create a marker on this layer with the given screen position and no
-  # tail.
-  #
-  # See the documentation for {TextEditor::markScreenPosition}
-  markScreenPosition: (screenPosition, options) ->
-    bufferPosition = @displayBuffer.bufferPositionForScreenPosition(screenPosition)
-    @markBufferPosition(bufferPosition, options)
-
-  ###
-  Section: Event Subscription
-  ###
-
-  # Public: Subscribe to be notified asynchronously whenever markers are
-  # created, updated, or destroyed on this layer. *Prefer this method for
-  # optimal performance when interacting with layers that could contain large
-  # numbers of markers.*
-  #
-  # * `callback` A {Function} that will be called with no arguments when changes
-  #   occur on this layer.
-  #
-  # Subscribers are notified once, asynchronously when any number of changes
-  # occur in a given tick of the event loop. You should re-query the layer
-  # to determine the state of markers in which you're interested in. It may
-  # be counter-intuitive, but this is much more efficient than subscribing to
-  # events on individual markers, which are expensive to deliver.
-  #
-  # Returns a {Disposable}.
-  onDidUpdate: (callback) ->
-    @bufferMarkerLayer.onDidUpdate(callback)
-
-  # Public: Subscribe to be notified synchronously whenever markers are created
-  # on this layer. *Avoid this method for optimal performance when interacting
-  # with layers that could contain large numbers of markers.*
-  #
-  # * `callback` A {Function} that will be called with a {TextEditorMarker}
-  #   whenever a new marker is created.
-  #
-  # You should prefer {onDidUpdate} when synchronous notifications aren't
-  # absolutely necessary.
-  #
-  # Returns a {Disposable}.
-  onDidCreateMarker: (callback) ->
-    @bufferMarkerLayer.onDidCreateMarker (bufferMarker) =>
-      callback(@getMarker(bufferMarker.id))
-
-  # Public: Subscribe to be notified synchronously when this layer is destroyed.
-  #
-  # Returns a {Disposable}.
-  onDidDestroy: (callback) ->
-    @bufferMarkerLayer.onDidDestroy(callback)
-
-  ###
-  Section: Private
-  ###
-
-  refreshMarkerScreenPositions: ->
-    for marker in @getMarkers()
-      marker.notifyObservers(textChanged: false)
-    return
-
-  didDestroyMarker: (marker) ->
-    delete @markersById[marker.id]
-
-  translateToBufferMarkerParams: (params) ->
-    bufferMarkerParams = {}
-    for key, value of params
-      switch key
-        when 'startBufferPosition'
-          key = 'startPosition'
-        when 'endBufferPosition'
-          key = 'endPosition'
-        when 'startScreenPosition'
-          key = 'startPosition'
-          value = @displayBuffer.bufferPositionForScreenPosition(value)
-        when 'endScreenPosition'
-          key = 'endPosition'
-          value = @displayBuffer.bufferPositionForScreenPosition(value)
-        when 'startBufferRow'
-          key = 'startRow'
-        when 'endBufferRow'
-          key = 'endRow'
-        when 'startScreenRow'
-          key = 'startRow'
-          value = @displayBuffer.bufferRowForScreenRow(value)
-        when 'endScreenRow'
-          key = 'endRow'
-          value = @displayBuffer.bufferRowForScreenRow(value)
-        when 'intersectsBufferRowRange'
-          key = 'intersectsRowRange'
-        when 'intersectsScreenRowRange'
-          key = 'intersectsRowRange'
-          [startRow, endRow] = value
-          value = [@displayBuffer.bufferRowForScreenRow(startRow), @displayBuffer.bufferRowForScreenRow(endRow)]
-        when 'containsBufferRange'
-          key = 'containsRange'
-        when 'containsBufferPosition'
-          key = 'containsPosition'
-        when 'containedInBufferRange'
-          key = 'containedInRange'
-        when 'containedInScreenRange'
-          key = 'containedInRange'
-          value = @displayBuffer.bufferRangeForScreenRange(value)
-        when 'intersectsBufferRange'
-          key = 'intersectsRange'
-        when 'intersectsScreenRange'
-          key = 'intersectsRange'
-          value = @displayBuffer.bufferRangeForScreenRange(value)
-      bufferMarkerParams[key] = value
-
-    bufferMarkerParams

src/text-editor-marker.coffee

@@ -1,371 +0,0 @@
-_ = require 'underscore-plus'
-{CompositeDisposable, Emitter} = require 'event-kit'
-
-# Essential: Represents a buffer annotation that remains logically stationary
-# even as the buffer changes. This is used to represent cursors, folds, snippet
-# targets, misspelled words, and anything else that needs to track a logical
-# location in the buffer over time.
-#
-# ### TextEditorMarker Creation
-#
-# Use {TextEditor::markBufferRange} rather than creating Markers directly.
-#
-# ### Head and Tail
-#
-# Markers always have a *head* and sometimes have a *tail*. If you think of a
-# marker as an editor selection, the tail is the part that's stationary and the
-# head is the part that moves when the mouse is moved. A marker without a tail
-# always reports an empty range at the head position. A marker with a head position
-# greater than the tail is in a "normal" orientation. If the head precedes the
-# tail the marker is in a "reversed" orientation.
-#
-# ### Validity
-#
-# Markers are considered *valid* when they are first created. Depending on the
-# invalidation strategy you choose, certain changes to the buffer can cause a
-# marker to become invalid, for example if the text surrounding the marker is
-# deleted. The strategies, in order of descending fragility:
-#
-# * __never__: The marker is never marked as invalid. This is a good choice for
-#   markers representing selections in an editor.
-# * __surround__: The marker is invalidated by changes that completely surround it.
-# * __overlap__: The marker is invalidated by changes that surround the
-#   start or end of the marker. This is the default.
-# * __inside__: The marker is invalidated by changes that extend into the
-#   inside of the marker. Changes that end at the marker's start or
-#   start at the marker's end do not invalidate the marker.
-# * __touch__: The marker is invalidated by a change that touches the marked
-#   region in any way, including changes that end at the marker's
-#   start or start at the marker's end. This is the most fragile strategy.
-#
-# See {TextEditor::markBufferRange} for usage.
-module.exports =
-class TextEditorMarker
-  bufferMarkerSubscription: null
-  oldHeadBufferPosition: null
-  oldHeadScreenPosition: null
-  oldTailBufferPosition: null
-  oldTailScreenPosition: null
-  wasValid: true
-  hasChangeObservers: false
-
-  ###
-  Section: Construction and Destruction
-  ###
-
-  constructor: (@layer, @bufferMarker) ->
-    {@displayBuffer} = @layer
-    @emitter = new Emitter
-    @disposables = new CompositeDisposable
-    @id = @bufferMarker.id
-
-    @disposables.add @bufferMarker.onDidDestroy => @destroyed()
-
-  # Essential: Destroys the marker, causing it to emit the 'destroyed' event. Once
-  # destroyed, a marker cannot be restored by undo/redo operations.
-  destroy: ->
-    @bufferMarker.destroy()
-    @disposables.dispose()
-
-  # Essential: Creates and returns a new {TextEditorMarker} with the same properties as
-  # this marker.
-  #
-  # {Selection} markers (markers with a custom property `type: "selection"`)
-  # should be copied with a different `type` value, for example with
-  # `marker.copy({type: null})`. Otherwise, the new marker's selection will
-  # be merged with this marker's selection, and a `null` value will be
-  # returned.
-  #
-  # * `properties` (optional) {Object} properties to associate with the new
-  # marker. The new marker's properties are computed by extending this marker's
-  # properties with `properties`.
-  #
-  # Returns a {TextEditorMarker}.
-  copy: (properties) ->
-    @layer.getMarker(@bufferMarker.copy(properties).id)
-
-  ###
-  Section: Event Subscription
-  ###
-
-  # Essential: Invoke the given callback when the state of the marker changes.
-  #
-  # * `callback` {Function} to be called when the marker changes.
-  #   * `event` {Object} with the following keys:
-  #     * `oldHeadBufferPosition` {Point} representing the former head buffer position
-  #     * `newHeadBufferPosition` {Point} representing the new head buffer position
-  #     * `oldTailBufferPosition` {Point} representing the former tail buffer position
-  #     * `newTailBufferPosition` {Point} representing the new tail buffer position
-  #     * `oldHeadScreenPosition` {Point} representing the former head screen position
-  #     * `newHeadScreenPosition` {Point} representing the new head screen position
-  #     * `oldTailScreenPosition` {Point} representing the former tail screen position
-  #     * `newTailScreenPosition` {Point} representing the new tail screen position
-  #     * `wasValid` {Boolean} indicating whether the marker was valid before the change
-  #     * `isValid` {Boolean} indicating whether the marker is now valid
-  #     * `hadTail` {Boolean} indicating whether the marker had a tail before the change
-  #     * `hasTail` {Boolean} indicating whether the marker now has a tail
-  #     * `oldProperties` {Object} containing the marker's custom properties before the change.
-  #     * `newProperties` {Object} containing the marker's custom properties after the change.
-  #     * `textChanged` {Boolean} indicating whether this change was caused by a textual change
-  #       to the buffer or whether the marker was manipulated directly via its public API.
-  #
-  # Returns a {Disposable} on which `.dispose()` can be called to unsubscribe.
-  onDidChange: (callback) ->
-    unless @hasChangeObservers
-      @oldHeadBufferPosition = @getHeadBufferPosition()
-      @oldHeadScreenPosition = @getHeadScreenPosition()
-      @oldTailBufferPosition = @getTailBufferPosition()
-      @oldTailScreenPosition = @getTailScreenPosition()
-      @wasValid = @isValid()
-      @disposables.add @bufferMarker.onDidChange (event) => @notifyObservers(event)
-      @hasChangeObservers = true
-    @emitter.on 'did-change', callback
-
-  # Essential: Invoke the given callback when the marker is destroyed.
-  #
-  # * `callback` {Function} to be called when the marker is destroyed.
-  #
-  # Returns a {Disposable} on which `.dispose()` can be called to unsubscribe.
-  onDidDestroy: (callback) ->
-    @emitter.on 'did-destroy', callback
-
-  ###
-  Section: TextEditorMarker Details
-  ###
-
-  # Essential: Returns a {Boolean} indicating whether the marker is valid. Markers can be
-  # invalidated when a region surrounding them in the buffer is changed.
-  isValid: ->
-    @bufferMarker.isValid()
-
-  # Essential: Returns a {Boolean} indicating whether the marker has been destroyed. A marker
-  # can be invalid without being destroyed, in which case undoing the invalidating
-  # operation would restore the marker. Once a marker is destroyed by calling
-  # {TextEditorMarker::destroy}, no undo/redo operation can ever bring it back.
-  isDestroyed: ->
-    @bufferMarker.isDestroyed()
-
-  # Essential: Returns a {Boolean} indicating whether the head precedes the tail.
-  isReversed: ->
-    @bufferMarker.isReversed()
-
-  # Essential: Get the invalidation strategy for this marker.
-  #
-  # Valid values include: `never`, `surround`, `overlap`, `inside`, and `touch`.
-  #
-  # Returns a {String}.
-  getInvalidationStrategy: ->
-    @bufferMarker.getInvalidationStrategy()
-
-  # Essential: Returns an {Object} containing any custom properties associated with
-  # the marker.
-  getProperties: ->
-    @bufferMarker.getProperties()
-
-  # Essential: Merges an {Object} containing new properties into the marker's
-  # existing properties.
-  #
-  # * `properties` {Object}
-  setProperties: (properties) ->
-    @bufferMarker.setProperties(properties)
-
-  matchesProperties: (attributes) ->
-    attributes = @layer.translateToBufferMarkerParams(attributes)
-    @bufferMarker.matchesParams(attributes)
-
-  ###
-  Section: Comparing to other markers
-  ###
-
-  # Essential: Returns a {Boolean} indicating whether this marker is equivalent to
-  # another marker, meaning they have the same range and options.
-  #
-  # * `other` {TextEditorMarker} other marker
-  isEqual: (other) ->
-    return false unless other instanceof @constructor
-    @bufferMarker.isEqual(other.bufferMarker)
-
-  # Essential: Compares this marker to another based on their ranges.
-  #
-  # * `other` {TextEditorMarker}
-  #
-  # Returns a {Number}
-  compare: (other) ->
-    @bufferMarker.compare(other.bufferMarker)
-
-  ###
-  Section: Managing the marker's range
-  ###
-
-  # Essential: Gets the buffer range of the display marker.
-  #
-  # Returns a {Range}.
-  getBufferRange: ->
-    @bufferMarker.getRange()
-
-  # Essential: Modifies the buffer range of the display marker.
-  #
-  # * `bufferRange` The new {Range} to use
-  # * `properties` (optional) {Object} properties to associate with the marker.
-  #   * `reversed` {Boolean} If true, the marker will to be in a reversed orientation.
-  setBufferRange: (bufferRange, properties) ->
-    @bufferMarker.setRange(bufferRange, properties)
-
-  # Essential: Gets the screen range of the display marker.
-  #
-  # Returns a {Range}.
-  getScreenRange: ->
-    @displayBuffer.screenRangeForBufferRange(@getBufferRange(), wrapAtSoftNewlines: true)
-
-  # Essential: Modifies the screen range of the display marker.
-  #
-  # * `screenRange` The new {Range} to use
-  # * `properties` (optional) {Object} properties to associate with the marker.
-  #   * `reversed` {Boolean} If true, the marker will to be in a reversed orientation.
-  setScreenRange: (screenRange, options) ->
-    @setBufferRange(@displayBuffer.bufferRangeForScreenRange(screenRange), options)
-
-  # Essential: Retrieves the buffer position of the marker's start. This will always be
-  # less than or equal to the result of {TextEditorMarker::getEndBufferPosition}.
-  #
-  # Returns a {Point}.
-  getStartBufferPosition: ->
-    @bufferMarker.getStartPosition()
-
-  # Essential: Retrieves the screen position of the marker's start. This will always be
-  # less than or equal to the result of {TextEditorMarker::getEndScreenPosition}.
-  #
-  # Returns a {Point}.
-  getStartScreenPosition: ->
-    @displayBuffer.screenPositionForBufferPosition(@getStartBufferPosition(), wrapAtSoftNewlines: true)
-
-  # Essential: Retrieves the buffer position of the marker's end. This will always be
-  # greater than or equal to the result of {TextEditorMarker::getStartBufferPosition}.
-  #
-  # Returns a {Point}.
-  getEndBufferPosition: ->
-    @bufferMarker.getEndPosition()
-
-  # Essential: Retrieves the screen position of the marker's end. This will always be
-  # greater than or equal to the result of {TextEditorMarker::getStartScreenPosition}.
-  #
-  # Returns a {Point}.
-  getEndScreenPosition: ->
-    @displayBuffer.screenPositionForBufferPosition(@getEndBufferPosition(), wrapAtSoftNewlines: true)
-
-  # Extended: Retrieves the buffer position of the marker's head.
-  #
-  # Returns a {Point}.
-  getHeadBufferPosition: ->
-    @bufferMarker.getHeadPosition()
-
-  # Extended: Sets the buffer position of the marker's head.
-  #
-  # * `bufferPosition` The new {Point} to use
-  # * `properties` (optional) {Object} properties to associate with the marker.
-  setHeadBufferPosition: (bufferPosition, properties) ->
-    @bufferMarker.setHeadPosition(bufferPosition, properties)
-
-  # Extended: Retrieves the screen position of the marker's head.
-  #
-  # Returns a {Point}.
-  getHeadScreenPosition: ->
-    @displayBuffer.screenPositionForBufferPosition(@getHeadBufferPosition(), wrapAtSoftNewlines: true)
-
-  # Extended: Sets the screen position of the marker's head.
-  #
-  # * `screenPosition` The new {Point} to use
-  # * `properties` (optional) {Object} properties to associate with the marker.
-  setHeadScreenPosition: (screenPosition, properties) ->
-    @setHeadBufferPosition(@displayBuffer.bufferPositionForScreenPosition(screenPosition, properties))
-
-  # Extended: Retrieves the buffer position of the marker's tail.
-  #
-  # Returns a {Point}.
-  getTailBufferPosition: ->
-    @bufferMarker.getTailPosition()
-
-  # Extended: Sets the buffer position of the marker's tail.
-  #
-  # * `bufferPosition` The new {Point} to use
-  # * `properties` (optional) {Object} properties to associate with the marker.
-  setTailBufferPosition: (bufferPosition) ->
-    @bufferMarker.setTailPosition(bufferPosition)
-
-  # Extended: Retrieves the screen position of the marker's tail.
-  #
-  # Returns a {Point}.
-  getTailScreenPosition: ->
-    @displayBuffer.screenPositionForBufferPosition(@getTailBufferPosition(), wrapAtSoftNewlines: true)
-
-  # Extended: Sets the screen position of the marker's tail.
-  #
-  # * `screenPosition` The new {Point} to use
-  # * `properties` (optional) {Object} properties to associate with the marker.
-  setTailScreenPosition: (screenPosition, options) ->
-    @setTailBufferPosition(@displayBuffer.bufferPositionForScreenPosition(screenPosition, options))
-
-  # Extended: Returns a {Boolean} indicating whether the marker has a tail.
-  hasTail: ->
-    @bufferMarker.hasTail()
-
-  # Extended: Plants the marker's tail at the current head position. After calling
-  # the marker's tail position will be its head position at the time of the
-  # call, regardless of where the marker's head is moved.
-  #
-  # * `properties` (optional) {Object} properties to associate with the marker.
-  plantTail: ->
-    @bufferMarker.plantTail()
-
-  # Extended: Removes the marker's tail. After calling the marker's head position
-  # will be reported as its current tail position until the tail is planted
-  # again.
-  #
-  # * `properties` (optional) {Object} properties to associate with the marker.
-  clearTail: (properties) ->
-    @bufferMarker.clearTail(properties)
-
-  ###
-  Section: Private utility methods
-  ###
-
-  # Returns a {String} representation of the marker
-  inspect: ->
-    "TextEditorMarker(id: #{@id}, bufferRange: #{@getBufferRange()})"
-
-  destroyed: ->
-    @layer.didDestroyMarker(this)
-    @emitter.emit 'did-destroy'
-    @emitter.dispose()
-
-  notifyObservers: ({textChanged}) ->
-    textChanged ?= false
-
-    newHeadBufferPosition = @getHeadBufferPosition()
-    newHeadScreenPosition = @getHeadScreenPosition()
-    newTailBufferPosition = @getTailBufferPosition()
-    newTailScreenPosition = @getTailScreenPosition()
-    isValid = @isValid()
-
-    return if isValid is @wasValid and
-      newHeadBufferPosition.isEqual(@oldHeadBufferPosition) and
-      newHeadScreenPosition.isEqual(@oldHeadScreenPosition) and
-      newTailBufferPosition.isEqual(@oldTailBufferPosition) and
-      newTailScreenPosition.isEqual(@oldTailScreenPosition)
-
-    changeEvent = {
-      @oldHeadScreenPosition, newHeadScreenPosition,
-      @oldTailScreenPosition, newTailScreenPosition,
-      @oldHeadBufferPosition, newHeadBufferPosition,
-      @oldTailBufferPosition, newTailBufferPosition,
-      textChanged,
-      isValid
-    }
-
-    @oldHeadBufferPosition = newHeadBufferPosition
-    @oldHeadScreenPosition = newHeadScreenPosition
-    @oldTailBufferPosition = newTailBufferPosition
-    @oldTailScreenPosition = newTailScreenPosition
-    @wasValid = isValid
-
-    @emitter.emit 'did-change', changeEvent

src/text-editor-presenter.coffee

@@ -16,6 +16,7 @@ class TextEditorPresenter
     {@model, @config, @lineTopIndex, scrollPastEnd} = params
     {@cursorBlinkPeriod, @cursorBlinkResumeDelay, @stoppedScrollingDelay, @tileSize} = params
     {@contentFrameWidth} = params
+    {@displayLayer} = @model
 
     @gutterWidth = 0
     @tileSize ?= 6
@@ -23,6 +24,7 @@ class TextEditorPresenter
     @realScrollLeft = @scrollLeft
     @disposables = new CompositeDisposable
     @emitter = new Emitter
+    @linesByScreenRow = new Map
     @visibleHighlights = {}
     @characterWidthsByScope = {}
     @lineDecorationsByScreenRow = {}
@@ -87,6 +89,8 @@ class TextEditorPresenter
     @updateCommonGutterState()
     @updateReflowState()
 
+    @updateLines()
+
     if @shouldUpdateDecorations
       @fetchDecorations()
       @updateLineDecorations()
@@ -106,6 +110,8 @@ class TextEditorPresenter
     @clearPendingScrollPosition()
     @updateRowsPerPage()
 
+    @updateLines()
+
     @updateFocusedState()
     @updateHeightState()
     @updateVerticalScrollState()
@@ -132,8 +138,11 @@ class TextEditorPresenter
     @shouldUpdateDecorations = true
 
   observeModel: ->
-    @disposables.add @model.onDidChange ({start, end, screenDelta}) =>
-      @spliceBlockDecorationsInRange(start, end, screenDelta)
+    @disposables.add @model.displayLayer.onDidChangeSync (changes) =>
+      for change in changes
+        startRow = change.start.row
+        endRow = startRow + change.oldExtent.row
+        @spliceBlockDecorationsInRange(startRow, endRow, change.newExtent.row - change.oldExtent.row)
       @shouldUpdateDecorations = true
       @emitDidUpdateState()
 
@@ -166,7 +175,6 @@ class TextEditorPresenter
 
     @scrollPastEnd = @config.get('editor.scrollPastEnd', configParams)
     @showLineNumbers = @config.get('editor.showLineNumbers', configParams)
-    @showIndentGuide = @config.get('editor.showIndentGuide', configParams)
 
     if @configDisposables?
       @configDisposables?.dispose()
@@ -175,10 +183,6 @@ class TextEditorPresenter
     @configDisposables = new CompositeDisposable
     @disposables.add(@configDisposables)
 
-    @configDisposables.add @config.onDidChange 'editor.showIndentGuide', configParams, ({newValue}) =>
-      @showIndentGuide = newValue
-
-      @emitDidUpdateState()
     @configDisposables.add @config.onDidChange 'editor.scrollPastEnd', configParams, ({newValue}) =>
       @scrollPastEnd = newValue
       @updateScrollHeight()
@@ -286,7 +290,6 @@ class TextEditorPresenter
     @state.content.width = Math.max(@contentWidth + @verticalScrollbarWidth, @contentFrameWidth)
     @state.content.scrollWidth = @scrollWidth
     @state.content.scrollLeft = @scrollLeft
-    @state.content.indentGuidesVisible = not @model.isMini() and @showIndentGuide
     @state.content.backgroundColor = if @model.isMini() then null else @backgroundColor
     @state.content.placeholderText = if @model.isEmpty() then @model.getPlaceholderText() else null
 
@@ -297,15 +300,15 @@ class TextEditorPresenter
     Math.max(0, Math.min(row, @model.getScreenLineCount()))
 
   getStartTileRow: ->
-    @constrainRow(@tileForRow(@startRow))
+    @constrainRow(@tileForRow(@startRow ? 0))
 
   getEndTileRow: ->
-    @constrainRow(@tileForRow(@endRow))
+    @constrainRow(@tileForRow(@endRow ? 0))
 
   isValidScreenRow: (screenRow) ->
     screenRow >= 0 and screenRow < @model.getScreenLineCount()
 
-  getScreenRows: ->
+  getScreenRowsToRender: ->
     startRow = @getStartTileRow()
     endRow = @constrainRow(@getEndTileRow() + @tileSize)
 
@@ -320,6 +323,22 @@ class TextEditorPresenter
     screenRows.sort (a, b) -> a - b
     _.uniq(screenRows, true)
 
+  getScreenRangesToRender: ->
+    screenRows = @getScreenRowsToRender()
+    screenRows.push(Infinity) # makes the loop below inclusive
+
+    startRow = screenRows[0]
+    endRow = startRow - 1
+    screenRanges = []
+    for row in screenRows
+      if row is endRow + 1
+        endRow++
+      else
+        screenRanges.push([startRow, endRow])
+        startRow = endRow = row
+
+    screenRanges
+
   setScreenRowsToMeasure: (screenRows) ->
     return if not screenRows? or screenRows.length is 0
 
@@ -332,7 +351,7 @@ class TextEditorPresenter
   updateTilesState: ->
     return unless @startRow? and @endRow? and @lineHeight?
 
-    screenRows = @getScreenRows()
+    screenRows = @getScreenRowsToRender()
     visibleTiles = {}
     startRow = screenRows[0]
     endRow = screenRows[screenRows.length - 1]
@@ -375,7 +394,7 @@ class TextEditorPresenter
       visibleTiles[tileStartRow] = true
       zIndex++
 
-    if @mouseWheelScreenRow? and @model.tokenizedLineForScreenRow(@mouseWheelScreenRow)?
+    if @mouseWheelScreenRow? and 0 <= @mouseWheelScreenRow < @model.getScreenLineCount()
       mouseWheelTile = @tileForRow(@mouseWheelScreenRow)
 
       unless visibleTiles[mouseWheelTile]?
@@ -393,7 +412,7 @@ class TextEditorPresenter
     tileState.lines ?= {}
     visibleLineIds = {}
     for screenRow in screenRows
-      line = @model.tokenizedLineForScreenRow(screenRow)
+      line = @linesByScreenRow.get(screenRow)
       unless line?
         throw new Error("No line exists for row #{screenRow}. Last screen row: #{@model.getLastScreenRow()}")
 
@@ -411,18 +430,8 @@ class TextEditorPresenter
       else
         tileState.lines[line.id] =
           screenRow: screenRow
-          text: line.text
-          openScopes: line.openScopes
-          tags: line.tags
-          specialTokens: line.specialTokens
-          firstNonWhitespaceIndex: line.firstNonWhitespaceIndex
-          firstTrailingWhitespaceIndex: line.firstTrailingWhitespaceIndex
-          invisibles: line.invisibles
-          endOfLineInvisibles: line.endOfLineInvisibles
-          isOnlyWhitespace: line.isOnlyWhitespace()
-          indentLevel: line.indentLevel
-          tabLength: line.tabLength
-          fold: line.fold
+          lineText: line.lineText
+          tagCodes: line.tagCodes
           decorationClasses: @lineDecorationClassesForRow(screenRow)
           precedingBlockDecorations: precedingBlockDecorations
           followingBlockDecorations: followingBlockDecorations
@@ -618,7 +627,7 @@ class TextEditorPresenter
           softWrapped = false
 
         screenRow = startRow + i
-        line = @model.tokenizedLineForScreenRow(screenRow)
+        lineId = @linesByScreenRow.get(screenRow).id
         decorationClasses = @lineNumberDecorationClassesForRow(screenRow)
         blockDecorationsBeforeCurrentScreenRowHeight = @lineTopIndex.pixelPositionAfterBlocksForRow(screenRow) - @lineTopIndex.pixelPositionBeforeBlocksForRow(screenRow)
         blockDecorationsHeight = blockDecorationsBeforeCurrentScreenRowHeight
@@ -626,8 +635,8 @@ class TextEditorPresenter
           blockDecorationsAfterPreviousScreenRowHeight = @lineTopIndex.pixelPositionBeforeBlocksForRow(screenRow) - @lineHeight - @lineTopIndex.pixelPositionAfterBlocksForRow(screenRow - 1)
           blockDecorationsHeight += blockDecorationsAfterPreviousScreenRowHeight
 
-        tileState.lineNumbers[line.id] = {screenRow, bufferRow, softWrapped, decorationClasses, foldable, blockDecorationsHeight}
-        visibleLineNumberIds[line.id] = true
+        tileState.lineNumbers[lineId] = {screenRow, bufferRow, softWrapped, decorationClasses, foldable, blockDecorationsHeight}
+        visibleLineNumberIds[lineId] = true
 
     for id of tileState.lineNumbers
       delete tileState.lineNumbers[id] unless visibleLineNumberIds[id]
@@ -687,9 +696,7 @@ class TextEditorPresenter
   updateHorizontalDimensions: ->
     if @baseCharacterWidth?
       oldContentWidth = @contentWidth
-      rightmostPosition = Point(@model.getLongestScreenRow(), @model.getMaxScreenLineLength())
-      if @model.tokenizedLineForScreenRow(rightmostPosition.row)?.isSoftWrapped()
-        rightmostPosition = @model.clipScreenPosition(rightmostPosition)
+      rightmostPosition = @model.getRightmostScreenPosition()
       @contentWidth = @pixelPositionForScreenPosition(rightmostPosition).left
       @contentWidth += @scrollLeft
       @contentWidth += 1 unless @model.isSoftWrapped() # account for cursor width
@@ -1057,6 +1064,16 @@ class TextEditorPresenter
     rect.height = Math.round(rect.height)
     rect
 
+  updateLines: ->
+    @linesByScreenRow.clear()
+
+    for [startRow, endRow] in @getScreenRangesToRender()
+      for line, index in @displayLayer.getScreenLines(startRow, endRow + 1)
+        @linesByScreenRow.set(startRow + index, line)
+
+  lineIdForScreenRow: (screenRow) ->
+    @linesByScreenRow.get(screenRow)?.id
+
   fetchDecorations: ->
     return unless 0 <= @startRow <= @endRow <= Infinity
     @decorations = @model.decorationsStateForScreenRowRange(@startRow, @endRow - 1)
@@ -1104,9 +1121,9 @@ class TextEditorPresenter
     @customGutterDecorationsByGutterName = {}
 
     for decorationId, decorationState of @decorations
-      {properties, screenRange, rangeIsReversed} = decorationState
+      {properties, bufferRange, screenRange, rangeIsReversed} = decorationState
       if Decoration.isType(properties, 'line') or Decoration.isType(properties, 'line-number')
-        @addToLineDecorationCaches(decorationId, properties, screenRange, rangeIsReversed)
+        @addToLineDecorationCaches(decorationId, properties, bufferRange, screenRange, rangeIsReversed)
 
       else if Decoration.isType(properties, 'gutter') and properties.gutterName?
         @customGutterDecorationsByGutterName[properties.gutterName] ?= {}
@@ -1127,7 +1144,7 @@ class TextEditorPresenter
 
     return
 
-  addToLineDecorationCaches: (decorationId, properties, screenRange, rangeIsReversed) ->
+  addToLineDecorationCaches: (decorationId, properties, bufferRange, screenRange, rangeIsReversed) ->
     if screenRange.isEmpty()
       return if properties.onlyNonEmpty
     else
@@ -1135,21 +1152,28 @@ class TextEditorPresenter
       omitLastRow = screenRange.end.column is 0
 
     if rangeIsReversed
-      headPosition = screenRange.start
+      headScreenPosition = screenRange.start
+      headBufferPosition = bufferRange.start
     else
-      headPosition = screenRange.end
+      headScreenPosition = screenRange.end
+      headBufferPosition = bufferRange.end
 
-    for row in [screenRange.start.row..screenRange.end.row] by 1
-      continue if properties.onlyHead and row isnt headPosition.row
-      continue if omitLastRow and row is screenRange.end.row
+    if properties.class is 'folded' and Decoration.isType(properties, 'line-number')
+      screenRow = @model.screenRowForBufferRow(headBufferPosition.row)
+      @lineNumberDecorationsByScreenRow[screenRow] ?= {}
+      @lineNumberDecorationsByScreenRow[screenRow][decorationId] = properties
+    else
+      for row in [screenRange.start.row..screenRange.end.row] by 1
+        continue if properties.onlyHead and row isnt headScreenPosition.row
+        continue if omitLastRow and row is screenRange.end.row
 
-      if Decoration.isType(properties, 'line')
-        @lineDecorationsByScreenRow[row] ?= {}
-        @lineDecorationsByScreenRow[row][decorationId] = properties
+        if Decoration.isType(properties, 'line')
+          @lineDecorationsByScreenRow[row] ?= {}
+          @lineDecorationsByScreenRow[row][decorationId] = properties
 
-      if Decoration.isType(properties, 'line-number')
-        @lineNumberDecorationsByScreenRow[row] ?= {}
-        @lineNumberDecorationsByScreenRow[row][decorationId] = properties
+        if Decoration.isType(properties, 'line-number')
+          @lineNumberDecorationsByScreenRow[row] ?= {}
+          @lineNumberDecorationsByScreenRow[row][decorationId] = properties
 
     return
 
@@ -1529,5 +1553,11 @@ class TextEditorPresenter
   isRowVisible: (row) ->
     @startRow <= row < @endRow
 
-  lineIdForScreenRow: (screenRow) ->
-    @model.tokenizedLineForScreenRow(screenRow)?.id
+  isOpenTagCode: (tagCode) ->
+    @displayLayer.isOpenTagCode(tagCode)
+
+  isCloseTagCode: (tagCode) ->
+    @displayLayer.isCloseTagCode(tagCode)
+
+  tagForCode: (tagCode) ->
+    @displayLayer.tagForCode(tagCode)

src/text-utils.coffee

@@ -94,6 +94,13 @@ isCJKCharacter = (character) ->
   isHalfWidthCharacter(character) or
   isKoreanCharacter(character)
 
+isWordStart = (previousCharacter, character) ->
+  (previousCharacter is ' ' or previousCharacter is '\t') and
+  (character isnt ' '  and character isnt '\t')
+
+isWrapBoundary = (previousCharacter, character) ->
+  isWordStart(previousCharacter, character) or isCJKCharacter(character)
+
 # Does the given string contain at least surrogate pair, variation sequence,
 # or combined character?
 #
@@ -107,4 +114,8 @@ hasPairedCharacter = (string) ->
     index++
   false
 
-module.exports = {isPairedCharacter, hasPairedCharacter, isDoubleWidthCharacter, isHalfWidthCharacter, isKoreanCharacter, isCJKCharacter}
+module.exports = {
+  isPairedCharacter, hasPairedCharacter,
+  isDoubleWidthCharacter, isHalfWidthCharacter, isKoreanCharacter,
+  isWrapBoundary
+}

src/token-iterator.coffee

@@ -1,106 +1,57 @@
-{SoftTab, HardTab, PairedCharacter, SoftWrapIndent} = require './special-token-symbols'
-{isDoubleWidthCharacter, isHalfWidthCharacter, isKoreanCharacter} = require './text-utils'
-
 module.exports =
 class TokenIterator
-  constructor: ({@grammarRegistry}, line, enableScopes) ->
-    @reset(line, enableScopes) if line?
+  constructor: ({@grammarRegistry}, line) ->
+    @reset(line) if line?
 
-  reset: (@line, @enableScopes=true) ->
+  reset: (@line) ->
     @index = null
-    @bufferStart = @line.startBufferColumn
-    @bufferEnd = @bufferStart
-    @screenStart = 0
-    @screenEnd = 0
-    @resetScopes() if @enableScopes
+    @startColumn = 0
+    @endColumn = 0
+    @scopes = @line.openScopes.map (id) => @grammarRegistry.scopeForId(id)
+    @scopeStarts = @scopes.slice()
+    @scopeEnds = []
     this
 
   next: ->
     {tags} = @line
 
     if @index?
+      @startColumn = @endColumn
+      @scopeEnds.length = 0
+      @scopeStarts.length = 0
       @index++
-      @bufferStart = @bufferEnd
-      @screenStart = @screenEnd
-      @clearScopeStartsAndEnds() if @enableScopes
     else
       @index = 0
 
     while @index < tags.length
       tag = tags[@index]
       if tag < 0
-        @handleScopeForTag(tag) if @enableScopes
+        scope = @grammarRegistry.scopeForId(tag)
+        if tag % 2 is 0
+          if @scopeStarts[@scopeStarts.length - 1] is scope
+            @scopeStarts.pop()
+          else
+            @scopeEnds.push(scope)
+          @scopes.pop()
+        else
+          @scopeStarts.push(scope)
+          @scopes.push(scope)
         @index++
       else
-        if @isHardTab()
-          @screenEnd = @screenStart + tag
-          @bufferEnd = @bufferStart + 1
-        else if @isSoftWrapIndentation()
-          @screenEnd = @screenStart + tag
-          @bufferEnd = @bufferStart + 0
-        else
-          @screenEnd = @screenStart + tag
-          @bufferEnd = @bufferStart + tag
-
-        @text = @line.text.substring(@screenStart, @screenEnd)
+        @endColumn += tag
+        @text = @line.text.substring(@startColumn, @endColumn)
         return true
 
     false
 
-  resetScopes: ->
-    @scopes = @line.openScopes.map (id) => @grammarRegistry.scopeForId(id)
-    @scopeStarts = @scopes.slice()
-    @scopeEnds = []
-
-  clearScopeStartsAndEnds: ->
-    @scopeEnds.length = 0
-    @scopeStarts.length = 0
-
-  handleScopeForTag: (tag) ->
-    scope = @grammarRegistry.scopeForId(tag)
-    if tag % 2 is 0
-      if @scopeStarts[@scopeStarts.length - 1] is scope
-        @scopeStarts.pop()
-      else
-        @scopeEnds.push(scope)
-      @scopes.pop()
-    else
-      @scopeStarts.push(scope)
-      @scopes.push(scope)
-
-  getBufferStart: -> @bufferStart
-  getBufferEnd: -> @bufferEnd
-
-  getScreenStart: -> @screenStart
-  getScreenEnd: -> @screenEnd
+  getScopes: -> @scopes
 
   getScopeStarts: -> @scopeStarts
-  getScopeEnds: -> @scopeEnds
 
-  getScopes: -> @scopes
+  getScopeEnds: -> @scopeEnds
 
   getText: -> @text
 
-  isSoftTab: ->
-    @line.specialTokens[@index] is SoftTab
+  getBufferStart: -> @startColumn
 
-  isHardTab: ->
-    @line.specialTokens[@index] is HardTab
-
-  isSoftWrapIndentation: ->
-    @line.specialTokens[@index] is SoftWrapIndent
-
-  isPairedCharacter: ->
-    @line.specialTokens[@index] is PairedCharacter
-
-  hasDoubleWidthCharacterAt: (charIndex) ->
-    isDoubleWidthCharacter(@getText()[charIndex])
-
-  hasHalfWidthCharacterAt: (charIndex) ->
-    isHalfWidthCharacter(@getText()[charIndex])
-
-  hasKoreanCharacterAt: (charIndex) ->
-    isKoreanCharacter(@getText()[charIndex])
-
-  isAtomic: ->
-    @isSoftTab() or @isHardTab() or @isSoftWrapIndentation() or @isPairedCharacter()
+  getBufferEnd: -> @endColumn

src/token.coffee

@@ -7,41 +7,20 @@ WhitespaceRegex = /\S/
 module.exports =
 class Token
   value: null
-  hasPairedCharacter: false
   scopes: null
-  isAtomic: null
-  isHardTab: null
-  firstNonWhitespaceIndex: null
-  firstTrailingWhitespaceIndex: null
-  hasInvisibleCharacters: false
 
   constructor: (properties) ->
-    {@value, @scopes, @isAtomic, @isHardTab, @bufferDelta} = properties
-    {@hasInvisibleCharacters, @hasPairedCharacter, @isSoftWrapIndentation} = properties
-    @firstNonWhitespaceIndex = properties.firstNonWhitespaceIndex ? null
-    @firstTrailingWhitespaceIndex = properties.firstTrailingWhitespaceIndex ? null
-
-    @screenDelta = @value.length
-    @bufferDelta ?= @screenDelta
+    {@value, @scopes} = properties
 
   isEqual: (other) ->
     # TODO: scopes is deprecated. This is here for the sake of lang package tests
-    @value is other.value and _.isEqual(@scopes, other.scopes) and !!@isAtomic is !!other.isAtomic
+    @value is other.value and _.isEqual(@scopes, other.scopes)
 
   isBracket: ->
     /^meta\.brace\b/.test(_.last(@scopes))
 
-  isOnlyWhitespace: ->
-    not WhitespaceRegex.test(@value)
-
   matchesScopeSelector: (selector) ->
     targetClasses = selector.replace(StartDotRegex, '').split('.')
     _.any @scopes, (scope) ->
       scopeClasses = scope.split('.')
       _.isSubset(targetClasses, scopeClasses)
-
-  hasLeadingWhitespace: ->
-    @firstNonWhitespaceIndex? and @firstNonWhitespaceIndex > 0
-
-  hasTrailingWhitespace: ->
-    @firstTrailingWhitespaceIndex? and @firstTrailingWhitespaceIndex < @value.length

src/tokenized-buffer-iterator.coffee

@@ -0,0 +1,122 @@
+{Point} = require 'text-buffer'
+
+module.exports =
+class TokenizedBufferIterator
+  constructor: (@tokenizedBuffer, @grammarRegistry) ->
+    @openTags = null
+    @closeTags = null
+    @containingTags = null
+
+  seek: (position) ->
+    @openTags = []
+    @closeTags = []
+    @tagIndex = null
+
+    currentLine = @tokenizedBuffer.tokenizedLineForRow(position.row)
+    @currentTags = currentLine.tags
+    @currentLineOpenTags = currentLine.openScopes
+    @currentLineLength = currentLine.text.length
+    @containingTags = @currentLineOpenTags.map (id) => @grammarRegistry.scopeForId(id)
+    currentColumn = 0
+    for tag, index in @currentTags
+      if tag >= 0
+        if currentColumn >= position.column and @isAtTagBoundary()
+          @tagIndex = index
+          break
+        else
+          currentColumn += tag
+          @containingTags.pop() while @closeTags.shift()
+          @containingTags.push(tag) while tag = @openTags.shift()
+      else
+        scopeName = @grammarRegistry.scopeForId(tag)
+        if tag % 2 is 0
+          if @openTags.length > 0
+            @tagIndex = index
+            break
+          else
+            @closeTags.push(scopeName)
+        else
+          @openTags.push(scopeName)
+
+    @tagIndex ?= @currentTags.length
+    @position = Point(position.row, Math.min(@currentLineLength, currentColumn))
+    @containingTags.slice()
+
+  moveToSuccessor: ->
+    @containingTags.pop() for tag in @closeTags
+    @containingTags.push(tag) for tag in @openTags
+    @openTags = []
+    @closeTags = []
+
+    loop
+      if @tagIndex is @currentTags.length
+        if @isAtTagBoundary()
+          break
+        else
+          if @shouldMoveToNextLine
+            @moveToNextLine()
+            @openTags = @currentLineOpenTags.map (id) => @grammarRegistry.scopeForId(id)
+            @shouldMoveToNextLine = false
+          else if @nextLineHasMismatchedContainingTags()
+            @closeTags = @containingTags.slice().reverse()
+            @containingTags = []
+            @shouldMoveToNextLine = true
+          else
+            return false unless @moveToNextLine()
+      else
+        tag = @currentTags[@tagIndex]
+        if tag >= 0
+          if @isAtTagBoundary()
+            break
+          else
+            @position = Point(@position.row, Math.min(@currentLineLength, @position.column + @currentTags[@tagIndex]))
+        else
+          scopeName = @grammarRegistry.scopeForId(tag)
+          if tag % 2 is 0
+            if @openTags.length > 0
+              break
+            else
+              @closeTags.push(scopeName)
+          else
+            @openTags.push(scopeName)
+        @tagIndex++
+
+    true
+
+  getPosition: ->
+    @position
+
+  getCloseTags: ->
+    @closeTags.slice()
+
+  getOpenTags: ->
+    @openTags.slice()
+
+  ###
+  Section: Private Methods
+  ###
+
+  nextLineHasMismatchedContainingTags: ->
+    if line = @tokenizedBuffer.tokenizedLineForRow(@position.row + 1)
+      return true if line.openScopes.length isnt @containingTags.length
+
+      for i in [0...@containingTags.length] by 1
+        if @containingTags[i] isnt @grammarRegistry.scopeForId(line.openScopes[i])
+          return true
+      false
+    else
+      false
+
+  moveToNextLine: ->
+    @position = Point(@position.row + 1, 0)
+    if tokenizedLine = @tokenizedBuffer.tokenizedLineForRow(@position.row)
+      @currentTags = tokenizedLine.tags
+      @currentLineLength = tokenizedLine.text.length
+      @currentLineOpenTags = tokenizedLine.openScopes
+      @tagIndex = 0
+      true
+    else
+      false
+
+  isAtTagBoundary: ->
+    @closeTags.length > 0 or @openTags.length > 0

src/tokenized-buffer.coffee

@@ -7,6 +7,7 @@ TokenizedLine = require './tokenized-line'
 TokenIterator = require './token-iterator'
 Token = require './token'
 ScopeDescriptor = require './scope-descriptor'
+TokenizedBufferIterator = require './tokenized-buffer-iterator'
 
 module.exports =
 class TokenizedBuffer extends Model
@@ -34,7 +35,7 @@ class TokenizedBuffer extends Model
 
   constructor: (params) ->
     {
-      @buffer, @tabLength, @ignoreInvisibles, @largeFileMode, @config,
+      @buffer, @tabLength, @largeFileMode, @config,
       @grammarRegistry, @assert, grammarScopeName
     } = params
 
@@ -57,13 +58,24 @@ class TokenizedBuffer extends Model
   destroyed: ->
     @disposables.dispose()
 
+  buildIterator: ->
+    new TokenizedBufferIterator(this, @grammarRegistry)
+
+  getInvalidatedRanges: ->
+    if @invalidatedRange?
+      [@invalidatedRange]
+    else
+      []
+
+  onDidInvalidateRange: (fn) ->
+    @emitter.on 'did-invalidate-range', fn
+
   serialize: ->
     state = {
       deserializer: 'TokenizedBuffer'
       bufferPath: @buffer.getPath()
       bufferId: @buffer.getId()
       tabLength: @tabLength
-      ignoreInvisibles: @ignoreInvisibles
       largeFileMode: @largeFileMode
     }
     state.grammarScopeName = @grammar?.scopeName unless @buffer.getPath()
@@ -104,24 +116,14 @@ class TokenizedBuffer extends Model
     @grammarUpdateDisposable = @grammar.onDidUpdate => @retokenizeLines()
     @disposables.add(@grammarUpdateDisposable)
 
-    scopeOptions = {scope: @rootScopeDescriptor}
-    @configSettings =
-      tabLength: @config.get('editor.tabLength', scopeOptions)
-      invisibles: @config.get('editor.invisibles', scopeOptions)
-      showInvisibles: @config.get('editor.showInvisibles', scopeOptions)
+    @configSettings = {tabLength: @config.get('editor.tabLength', {scope: @rootScopeDescriptor})}
 
     if @configSubscriptions?
       @configSubscriptions.dispose()
       @disposables.remove(@configSubscriptions)
     @configSubscriptions = new CompositeDisposable
-    @configSubscriptions.add @config.onDidChange 'editor.tabLength', scopeOptions, ({newValue}) =>
+    @configSubscriptions.add @config.onDidChange 'editor.tabLength', {scope: @rootScopeDescriptor}, ({newValue}) =>
       @configSettings.tabLength = newValue
-      @retokenizeLines()
-    ['invisibles', 'showInvisibles'].forEach (key) =>
-      @configSubscriptions.add @config.onDidChange "editor.#{key}", scopeOptions, ({newValue}) =>
-        oldInvisibles = @getInvisiblesToShow()
-        @configSettings[key] = newValue
-        @retokenizeLines() unless _.isEqual(@getInvisiblesToShow(), oldInvisibles)
     @disposables.add(@configSubscriptions)
 
     @retokenizeLines()
@@ -162,13 +164,6 @@ class TokenizedBuffer extends Model
     return if tabLength is @tabLength
 
     @tabLength = tabLength
-    @retokenizeLines()
-
-  setIgnoreInvisibles: (ignoreInvisibles) ->
-    if ignoreInvisibles isnt @ignoreInvisibles
-      @ignoreInvisibles = ignoreInvisibles
-      if @configSettings.showInvisibles and @configSettings.invisibles?
-        @retokenizeLines()
 
   tokenizeInBackground: ->
     return if not @visible or @pendingChunk or not @isAlive()
@@ -211,6 +206,7 @@ class TokenizedBuffer extends Model
 
       event = {start: startRow, end: endRow, delta: 0}
       @emitter.emit 'did-change', event
+      @emitter.emit 'did-invalidate-range', Range(Point(startRow, 0), Point(endRow + 1, 0))
 
     if @firstInvalidRow()?
       @tokenizeInBackground()
@@ -261,26 +257,15 @@ class TokenizedBuffer extends Model
       newTokenizedLines = @buildTokenizedLinesForRows(start, end + delta, @stackForRow(start - 1), @openScopesForRow(start))
     _.spliceWithArray(@tokenizedLines, start, end - start + 1, newTokenizedLines)
 
-    start = @retokenizeWhitespaceRowsIfIndentLevelChanged(start - 1, -1)
-    end = @retokenizeWhitespaceRowsIfIndentLevelChanged(newRange.end.row + 1, 1) - delta
-
     newEndStack = @stackForRow(end + delta)
     if newEndStack and not _.isEqual(newEndStack, previousEndStack)
       @invalidateRow(end + delta + 1)
 
+    @invalidatedRange = Range(start, end)
+
     event = {start, end, delta, bufferChange: e}
     @emitter.emit 'did-change', event
 
-  retokenizeWhitespaceRowsIfIndentLevelChanged: (row, increment) ->
-    line = @tokenizedLineForRow(row)
-    if line?.isOnlyWhitespace() and @indentLevelForRow(row) isnt line.indentLevel
-      while line?.isOnlyWhitespace()
-        @tokenizedLines[row] = @buildTokenizedLineForRow(row, @stackForRow(row - 1), @openScopesForRow(row))
-        row += increment
-        line = @tokenizedLineForRow(row)
-
-    row - increment
-
   isFoldableAtRow: (row) ->
     if @largeFileMode
       false
@@ -345,26 +330,16 @@ class TokenizedBuffer extends Model
     openScopes = [@grammar.startIdForScope(@grammar.scopeName)]
     text = @buffer.lineForRow(row)
     tags = [text.length]
-    tabLength = @getTabLength()
-    indentLevel = @indentLevelForRow(row)
     lineEnding = @buffer.lineEndingForRow(row)
-    new TokenizedLine({openScopes, text, tags, tabLength, indentLevel, invisibles: @getInvisiblesToShow(), lineEnding, @tokenIterator})
+    new TokenizedLine({openScopes, text, tags, lineEnding, @tokenIterator})
 
   buildTokenizedLineForRow: (row, ruleStack, openScopes) ->
     @buildTokenizedLineForRowWithText(row, @buffer.lineForRow(row), ruleStack, openScopes)
 
   buildTokenizedLineForRowWithText: (row, text, ruleStack = @stackForRow(row - 1), openScopes = @openScopesForRow(row)) ->
     lineEnding = @buffer.lineEndingForRow(row)
-    tabLength = @getTabLength()
-    indentLevel = @indentLevelForRow(row)
     {tags, ruleStack} = @grammar.tokenizeLine(text, ruleStack, row is 0, false)
-    new TokenizedLine({openScopes, text, tags, ruleStack, tabLength, lineEnding, indentLevel, invisibles: @getInvisiblesToShow(), @tokenIterator})
-
-  getInvisiblesToShow: ->
-    if @configSettings.showInvisibles and not @ignoreInvisibles
-      @configSettings.invisibles
-    else
-      null
+    new TokenizedLine({openScopes, text, tags, ruleStack, lineEnding, @tokenIterator})
 
   tokenizedLineForRow: (bufferRow) ->
     if 0 <= bufferRow < @tokenizedLines.length
@@ -405,6 +380,7 @@ class TokenizedBuffer extends Model
                 filePath: @buffer.getPath()
                 fileContents: @buffer.getText()
               }
+            break
     scopes
 
   indentLevelForRow: (bufferRow) ->

src/tokenized-line.coffee

@@ -1,187 +1,18 @@
 _ = require 'underscore-plus'
 {isPairedCharacter, isCJKCharacter} = require './text-utils'
 Token = require './token'
-{SoftTab, HardTab, PairedCharacter, SoftWrapIndent} = require './special-token-symbols'
-
-NonWhitespaceRegex = /\S/
-LeadingWhitespaceRegex = /^\s*/
-TrailingWhitespaceRegex = /\s*$/
-RepeatedSpaceRegex = /[ ]/g
 CommentScopeRegex = /(\b|\.)comment/
-TabCharCode = 9
-SpaceCharCode = 32
-SpaceString = ' '
-TabStringsByLength = {
-  1: ' '
-  2: '  '
-  3: '   '
-  4: '    '
-}
 
 idCounter = 1
 
-getTabString = (length) ->
-  TabStringsByLength[length] ?= buildTabString(length)
-
-buildTabString = (length) ->
-  string = SpaceString
-  string += SpaceString for i in [1...length] by 1
-  string
-
 module.exports =
 class TokenizedLine
-  endOfLineInvisibles: null
-  lineIsWhitespaceOnly: false
-  firstNonWhitespaceIndex: 0
-
   constructor: (properties) ->
     @id = idCounter++
 
     return unless properties?
 
-    @specialTokens = {}
-    {@openScopes, @text, @tags, @lineEnding, @ruleStack, @tokenIterator} = properties
-    {@startBufferColumn, @fold, @tabLength, @indentLevel, @invisibles} = properties
-
-    @startBufferColumn ?= 0
-    @bufferDelta = @text.length
-
-    @transformContent()
-    @buildEndOfLineInvisibles() if @invisibles? and @lineEnding?
-
-  transformContent: ->
-    text = ''
-    bufferColumn = 0
-    screenColumn = 0
-    tokenIndex = 0
-    tokenOffset = 0
-    firstNonWhitespaceColumn = null
-    lastNonWhitespaceColumn = null
-
-    substringStart = 0
-    substringEnd = 0
-
-    while bufferColumn < @text.length
-      # advance to next token if we've iterated over its length
-      if tokenOffset is @tags[tokenIndex]
-        tokenIndex++
-        tokenOffset = 0
-
-      # advance to next token tag
-      tokenIndex++ while @tags[tokenIndex] < 0
-
-      charCode = @text.charCodeAt(bufferColumn)
-
-      # split out unicode surrogate pairs
-      if isPairedCharacter(@text, bufferColumn)
-        prefix = tokenOffset
-        suffix = @tags[tokenIndex] - tokenOffset - 2
-
-        i = tokenIndex
-        @tags.splice(i, 1)
-        @tags.splice(i++, 0, prefix) if prefix > 0
-        @tags.splice(i++, 0, 2)
-        @tags.splice(i, 0, suffix) if suffix > 0
-
-        firstNonWhitespaceColumn ?= screenColumn
-        lastNonWhitespaceColumn = screenColumn + 1
-
-        substringEnd += 2
-        screenColumn += 2
-        bufferColumn += 2
-
-        tokenIndex++ if prefix > 0
-        @specialTokens[tokenIndex] = PairedCharacter
-        tokenIndex++
-        tokenOffset = 0
-
-      # split out leading soft tabs
-      else if charCode is SpaceCharCode
-        if firstNonWhitespaceColumn?
-          substringEnd += 1
-        else
-          if (screenColumn + 1) % @tabLength is 0
-            suffix = @tags[tokenIndex] - @tabLength
-            if suffix >= 0
-              @specialTokens[tokenIndex] = SoftTab
-              @tags.splice(tokenIndex, 1, @tabLength)
-              @tags.splice(tokenIndex + 1, 0, suffix) if suffix > 0
-
-          if @invisibles?.space
-            if substringEnd > substringStart
-              text += @text.substring(substringStart, substringEnd)
-            substringStart = substringEnd
-            text += @invisibles.space
-            substringStart += 1
-
-          substringEnd += 1
-
-        screenColumn++
-        bufferColumn++
-        tokenOffset++
-
-      # expand hard tabs to the next tab stop
-      else if charCode is TabCharCode
-        if substringEnd > substringStart
-          text += @text.substring(substringStart, substringEnd)
-        substringStart = substringEnd
-
-        tabLength = @tabLength - (screenColumn % @tabLength)
-        if @invisibles?.tab
-          text += @invisibles.tab
-          text += getTabString(tabLength - 1) if tabLength > 1
-        else
-          text += getTabString(tabLength)
-
-        substringStart += 1
-        substringEnd += 1
-
-        prefix = tokenOffset
-        suffix = @tags[tokenIndex] - tokenOffset - 1
-
-        i = tokenIndex
-        @tags.splice(i, 1)
-        @tags.splice(i++, 0, prefix) if prefix > 0
-        @tags.splice(i++, 0, tabLength)
-        @tags.splice(i, 0, suffix) if suffix > 0
-
-        screenColumn += tabLength
-        bufferColumn++
-
-        tokenIndex++ if prefix > 0
-        @specialTokens[tokenIndex] = HardTab
-        tokenIndex++
-        tokenOffset = 0
-
-      # continue past any other character
-      else
-        firstNonWhitespaceColumn ?= screenColumn
-        lastNonWhitespaceColumn = screenColumn
-
-        substringEnd += 1
-        screenColumn++
-        bufferColumn++
-        tokenOffset++
-
-    if substringEnd > substringStart
-      unless substringStart is 0 and substringEnd is @text.length
-        text += @text.substring(substringStart, substringEnd)
-        @text = text
-    else
-      @text = text
-
-    @firstNonWhitespaceIndex = firstNonWhitespaceColumn
-    if lastNonWhitespaceColumn?
-      if lastNonWhitespaceColumn + 1 < @text.length
-        @firstTrailingWhitespaceIndex = lastNonWhitespaceColumn + 1
-        if @invisibles?.space
-          @text =
-            @text.substring(0, @firstTrailingWhitespaceIndex) +
-              @text.substring(@firstTrailingWhitespaceIndex)
-                .replace(RepeatedSpaceRegex, @invisibles.space)
-    else
-      @lineIsWhitespaceOnly = true
-      @firstTrailingWhitespaceIndex = 0
+    {@openScopes, @text, @tags, @ruleStack, @tokenIterator} = properties
 
   getTokenIterator: -> @tokenIterator.reset(this, arguments...)
 
@@ -190,285 +21,21 @@ class TokenizedLine
     tokens = []
 
     while iterator.next()
-      properties = {
+      tokens.push(new Token({
         value: iterator.getText()
         scopes: iterator.getScopes().slice()
-        isAtomic: iterator.isAtomic()
-        isHardTab: iterator.isHardTab()
-        hasPairedCharacter: iterator.isPairedCharacter()
-        isSoftWrapIndentation: iterator.isSoftWrapIndentation()
-      }
-
-      if iterator.isHardTab()
-        properties.bufferDelta = 1
-        properties.hasInvisibleCharacters = true if @invisibles?.tab
-
-      if iterator.getScreenStart() < @firstNonWhitespaceIndex
-        properties.firstNonWhitespaceIndex =
-          Math.min(@firstNonWhitespaceIndex, iterator.getScreenEnd()) - iterator.getScreenStart()
-        properties.hasInvisibleCharacters = true if @invisibles?.space
-
-      if @lineEnding? and iterator.getScreenEnd() > @firstTrailingWhitespaceIndex
-        properties.firstTrailingWhitespaceIndex =
-          Math.max(0, @firstTrailingWhitespaceIndex - iterator.getScreenStart())
-        properties.hasInvisibleCharacters = true if @invisibles?.space
-
-      tokens.push(new Token(properties))
+      }))
 
     tokens
 
-  copy: ->
-    copy = new TokenizedLine
-    copy.tokenIterator = @tokenIterator
-    copy.openScopes = @openScopes
-    copy.text = @text
-    copy.tags = @tags
-    copy.specialTokens = @specialTokens
-    copy.startBufferColumn = @startBufferColumn
-    copy.bufferDelta = @bufferDelta
-    copy.ruleStack = @ruleStack
-    copy.lineEnding = @lineEnding
-    copy.invisibles = @invisibles
-    copy.endOfLineInvisibles = @endOfLineInvisibles
-    copy.indentLevel = @indentLevel
-    copy.tabLength = @tabLength
-    copy.firstNonWhitespaceIndex = @firstNonWhitespaceIndex
-    copy.firstTrailingWhitespaceIndex = @firstTrailingWhitespaceIndex
-    copy.fold = @fold
-    copy
-
-  # This clips a given screen column to a valid column that's within the line
-  # and not in the middle of any atomic tokens.
-  #
-  # column - A {Number} representing the column to clip
-  # options - A hash with the key clip. Valid values for this key:
-  #           'closest' (default): clip to the closest edge of an atomic token.
-  #           'forward': clip to the forward edge.
-  #           'backward': clip to the backward edge.
-  #
-  # Returns a {Number} representing the clipped column.
-  clipScreenColumn: (column, options={}) ->
-    return 0 if @tags.length is 0
-
-    {clip} = options
-    column = Math.min(column, @getMaxScreenColumn())
-
-    tokenStartColumn = 0
-
-    iterator = @getTokenIterator()
-    while iterator.next()
-      break if iterator.getScreenEnd() > column
-
-    if iterator.isSoftWrapIndentation()
-      iterator.next() while iterator.isSoftWrapIndentation()
-      iterator.getScreenStart()
-    else if iterator.isAtomic() and iterator.getScreenStart() < column
-      if clip is 'forward'
-        iterator.getScreenEnd()
-      else if clip is 'backward'
-        iterator.getScreenStart()
-      else #'closest'
-        if column > ((iterator.getScreenStart() + iterator.getScreenEnd()) / 2)
-          iterator.getScreenEnd()
-        else
-          iterator.getScreenStart()
-    else
-      column
-
-  screenColumnForBufferColumn: (targetBufferColumn, options) ->
-    iterator = @getTokenIterator()
-    while iterator.next()
-      tokenBufferStart = iterator.getBufferStart()
-      tokenBufferEnd = iterator.getBufferEnd()
-      if tokenBufferStart <= targetBufferColumn < tokenBufferEnd
-        overshoot = targetBufferColumn - tokenBufferStart
-        return Math.min(
-          iterator.getScreenStart() + overshoot,
-          iterator.getScreenEnd()
-        )
-    iterator.getScreenEnd()
-
-  bufferColumnForScreenColumn: (targetScreenColumn) ->
-    iterator = @getTokenIterator()
-    while iterator.next()
-      tokenScreenStart = iterator.getScreenStart()
-      tokenScreenEnd = iterator.getScreenEnd()
-      if tokenScreenStart <= targetScreenColumn < tokenScreenEnd
-        overshoot = targetScreenColumn - tokenScreenStart
-        return Math.min(
-          iterator.getBufferStart() + overshoot,
-          iterator.getBufferEnd()
-        )
-    iterator.getBufferEnd()
-
-  getMaxScreenColumn: ->
-    if @fold
-      0
-    else
-      @text.length
-
-  getMaxBufferColumn: ->
-    @startBufferColumn + @bufferDelta
-
-  # Given a boundary column, finds the point where this line would wrap.
-  #
-  # maxColumn - The {Number} where you want soft wrapping to occur
-  #
-  # Returns a {Number} representing the `line` position where the wrap would take place.
-  # Returns `null` if a wrap wouldn't occur.
-  findWrapColumn: (maxColumn) ->
-    return unless maxColumn?
-    return unless @text.length > maxColumn
-
-    if /\s/.test(@text[maxColumn])
-      # search forward for the start of a word past the boundary
-      for column in [maxColumn..@text.length]
-        return column if /\S/.test(@text[column])
-
-      return @text.length
-    else if isCJKCharacter(@text[maxColumn])
-      maxColumn
-    else
-      # search backward for the start of the word on the boundary
-      for column in [maxColumn..@firstNonWhitespaceIndex]
-        if /\s/.test(@text[column]) or isCJKCharacter(@text[column])
-          return column + 1
-
-      return maxColumn
-
-  softWrapAt: (column, hangingIndent) ->
-    return [null, this] if column is 0
-
-    leftText = @text.substring(0, column)
-    rightText = @text.substring(column)
-
-    leftTags = []
-    rightTags = []
-
-    leftSpecialTokens = {}
-    rightSpecialTokens = {}
-
-    rightOpenScopes = @openScopes.slice()
-
-    screenColumn = 0
-
-    for tag, index in @tags
-      # tag represents a token
-      if tag >= 0
-        # token ends before the soft wrap column
-        if screenColumn + tag <= column
-          if specialToken = @specialTokens[index]
-            leftSpecialTokens[index] = specialToken
-          leftTags.push(tag)
-          screenColumn += tag
-
-        # token starts before and ends after the split column
-        else if screenColumn <= column
-          leftSuffix = column - screenColumn
-          rightPrefix = screenColumn + tag - column
-
-          leftTags.push(leftSuffix) if leftSuffix > 0
-
-          softWrapIndent = @indentLevel * @tabLength + (hangingIndent ? 0)
-          for i in [0...softWrapIndent] by 1
-            rightText = ' ' + rightText
-          remainingSoftWrapIndent = softWrapIndent
-          while remainingSoftWrapIndent > 0
-            indentToken = Math.min(remainingSoftWrapIndent, @tabLength)
-            rightSpecialTokens[rightTags.length] = SoftWrapIndent
-            rightTags.push(indentToken)
-            remainingSoftWrapIndent -= indentToken
-
-          rightTags.push(rightPrefix) if rightPrefix > 0
-
-          screenColumn += tag
-
-         # token is after split column
-        else
-          if specialToken = @specialTokens[index]
-            rightSpecialTokens[rightTags.length] = specialToken
-          rightTags.push(tag)
-
-      # tag represents the start of a scope
-      else if (tag % 2) is -1
-        if screenColumn < column
-          leftTags.push(tag)
-          rightOpenScopes.push(tag)
-        else
-          rightTags.push(tag)
-
-      # tag represents the end of a scope
-      else
-        if screenColumn <= column
-          leftTags.push(tag)
-          rightOpenScopes.pop()
-        else
-          rightTags.push(tag)
-
-    splitBufferColumn = @bufferColumnForScreenColumn(column)
-
-    leftFragment = new TokenizedLine
-    leftFragment.tokenIterator = @tokenIterator
-    leftFragment.openScopes = @openScopes
-    leftFragment.text = leftText
-    leftFragment.tags = leftTags
-    leftFragment.specialTokens = leftSpecialTokens
-    leftFragment.startBufferColumn = @startBufferColumn
-    leftFragment.bufferDelta = splitBufferColumn - @startBufferColumn
-    leftFragment.ruleStack = @ruleStack
-    leftFragment.invisibles = @invisibles
-    leftFragment.lineEnding = null
-    leftFragment.indentLevel = @indentLevel
-    leftFragment.tabLength = @tabLength
-    leftFragment.firstNonWhitespaceIndex = Math.min(column, @firstNonWhitespaceIndex)
-    leftFragment.firstTrailingWhitespaceIndex = Math.min(column, @firstTrailingWhitespaceIndex)
-
-    rightFragment = new TokenizedLine
-    rightFragment.tokenIterator = @tokenIterator
-    rightFragment.openScopes = rightOpenScopes
-    rightFragment.text = rightText
-    rightFragment.tags = rightTags
-    rightFragment.specialTokens = rightSpecialTokens
-    rightFragment.startBufferColumn = splitBufferColumn
-    rightFragment.bufferDelta = @startBufferColumn + @bufferDelta - splitBufferColumn
-    rightFragment.ruleStack = @ruleStack
-    rightFragment.invisibles = @invisibles
-    rightFragment.lineEnding = @lineEnding
-    rightFragment.indentLevel = @indentLevel
-    rightFragment.tabLength = @tabLength
-    rightFragment.endOfLineInvisibles = @endOfLineInvisibles
-    rightFragment.firstNonWhitespaceIndex = Math.max(softWrapIndent, @firstNonWhitespaceIndex - column + softWrapIndent)
-    rightFragment.firstTrailingWhitespaceIndex = Math.max(softWrapIndent, @firstTrailingWhitespaceIndex - column + softWrapIndent)
-
-    [leftFragment, rightFragment]
-
-  isSoftWrapped: ->
-    @lineEnding is null
-
-  isColumnInsideSoftWrapIndentation: (targetColumn) ->
-    targetColumn < @getSoftWrapIndentationDelta()
-
-  getSoftWrapIndentationDelta: ->
-    delta = 0
-    for tag, index in @tags
-      if tag >= 0
-        if @specialTokens[index] is SoftWrapIndent
-          delta += tag
-        else
-          break
-    delta
-
-  hasOnlySoftWrapIndentation: ->
-    @getSoftWrapIndentationDelta() is @text.length
-
   tokenAtBufferColumn: (bufferColumn) ->
     @tokens[@tokenIndexAtBufferColumn(bufferColumn)]
 
   tokenIndexAtBufferColumn: (bufferColumn) ->
-    delta = 0
+    column = 0
     for token, index in @tokens
-      delta += token.bufferDelta
-      return index if delta > bufferColumn
+      column += token.value.length
+      return index if column > bufferColumn
     index - 1
 
   tokenStartColumnForBufferColumn: (bufferColumn) ->
@@ -479,17 +46,6 @@ class TokenizedLine
       delta = nextDelta
     delta
 
-  buildEndOfLineInvisibles: ->
-    @endOfLineInvisibles = []
-    {cr, eol} = @invisibles
-
-    switch @lineEnding
-      when '\r\n'
-        @endOfLineInvisibles.push(cr) if cr
-        @endOfLineInvisibles.push(eol) if eol
-      when '\n'
-        @endOfLineInvisibles.push(eol) if eol
-
   isComment: ->
     return @isCommentLine if @isCommentLine?
 
@@ -505,9 +61,6 @@ class TokenizedLine
       break
     @isCommentLine
 
-  isOnlyWhitespace: ->
-    @lineIsWhitespaceOnly
-
   tokenAtIndex: (index) ->
     @tokens[index]