atom/src/tokenized-line.coffee

_ = require 'underscore-plus'
{isPairedCharacter} = require './text-utils'

NonWhitespaceRegex = /\S/
LeadingWhitespaceRegex = /^\s*/
TrailingWhitespaceRegex = /\s*$/
RepeatedSpaceRegex = /[ ]/g
idCounter = 1

module.exports =
class TokenizedLine
  endOfLineInvisibles: null
  lineIsWhitespaceOnly: false
  firstNonWhitespaceIndex: 0
  foldable: false

  constructor: ({tokens, @lineEnding, @ruleStack, @startBufferColumn, @fold, @tabLength, @indentLevel, @invisibles}) ->
    @startBufferColumn ?= 0
    @tokens = @breakOutAtomicTokens(tokens)
    @text = @buildText()
    @bufferDelta = @buildBufferDelta()
    @softWrapIndentationTokens = @getSoftWrapIndentationTokens()
    @softWrapIndentationDelta = @buildSoftWrapIndentationDelta()

    @id = idCounter++
    @markLeadingAndTrailingWhitespaceTokens()
    if @invisibles
      @substituteInvisibleCharacters()
      @buildEndOfLineInvisibles() if @lineEnding?

  buildText: ->
    text = ""
    text += token.value for token in @tokens
    text

  buildBufferDelta: ->
    delta = 0
    delta += token.bufferDelta for token in @tokens
    delta

  copy: ->
    new TokenizedLine({@tokens, @lineEnding, @ruleStack, @startBufferColumn, @fold})

  # This clips a given screen column to a valid column that's within the line
  # and not in the middle of any atomic tokens.
  #
  # column - A {Number} representing the column to clip
  # options - A hash with the key clip. Valid values for this key:
  #           'closest' (default): clip to the closest edge of an atomic token.
  #           'forward': clip to the forward edge.
  #           'backward': clip to the backward edge.
  #
  # Returns a {Number} representing the clipped column.
  clipScreenColumn: (column, options={}) ->
    return 0 if @tokens.length is 0

    {clip} = options
    column = Math.min(column, @getMaxScreenColumn())

    tokenStartColumn = 0
    for token in @tokens
      break if tokenStartColumn + token.screenDelta > column
      tokenStartColumn += token.screenDelta

    if @isColumnInsideSoftWrapIndentation(tokenStartColumn)
      @softWrapIndentationDelta
    else if token.isAtomic and tokenStartColumn < column
      if clip is 'forward'
        tokenStartColumn + token.screenDelta
      else if clip is 'backward'
        tokenStartColumn
      else #'closest'
        if column > tokenStartColumn + (token.screenDelta / 2)
          tokenStartColumn + token.screenDelta
        else
          tokenStartColumn
    else
      column

  screenColumnForBufferColumn: (bufferColumn, options) ->
    bufferColumn = bufferColumn - @startBufferColumn
    screenColumn = 0
    currentBufferColumn = 0
    for token in @tokens
      break if currentBufferColumn + token.bufferDelta > bufferColumn
      screenColumn += token.screenDelta
      currentBufferColumn += token.bufferDelta
    @clipScreenColumn(screenColumn + (bufferColumn - currentBufferColumn))

  bufferColumnForScreenColumn: (screenColumn, options) ->
    bufferColumn = @startBufferColumn
    currentScreenColumn = 0
    for token in @tokens
      break if currentScreenColumn + token.screenDelta > screenColumn
      bufferColumn += token.bufferDelta
      currentScreenColumn += token.screenDelta
    bufferColumn + (screenColumn - currentScreenColumn)

  getMaxScreenColumn: ->
    if @fold
      0
    else
      @text.length

  getMaxBufferColumn: ->
    @startBufferColumn + @bufferDelta

  # Given a boundary column, finds the point where this line would wrap.
  #
  # maxColumn - The {Number} where you want soft wrapping to occur
  #
  # Returns a {Number} representing the `line` position where the wrap would take place.
  # Returns `null` if a wrap wouldn't occur.
  findWrapColumn: (maxColumn) ->
    return unless maxColumn?
    return unless @text.length > maxColumn

    if /\s/.test(@text[maxColumn])
       # search forward for the start of a word past the boundary
      for column in [maxColumn..@text.length]
        return column if /\S/.test(@text[column])

      return @text.length
    else
      # search backward for the start of the word on the boundary
      for column in [maxColumn..@firstNonWhitespaceIndex]
        return column + 1 if /\s/.test(@text[column])

      return maxColumn

  buildSoftWrapIndentationTokens: (token, hangingIndent) ->
    totalIndentSpaces = (@indentLevel * @tabLength) + hangingIndent
    indentTokens = []
    while totalIndentSpaces > 0
      tokenLength = Math.min(@tabLength, totalIndentSpaces)
      indentToken = token.buildSoftWrapIndentationToken(tokenLength)
      indentTokens.push(indentToken)
      totalIndentSpaces -= tokenLength

    indentTokens

  softWrapAt: (column, hangingIndent) ->
    return [new TokenizedLine([], '', [0, 0], [0, 0]), this] if column is 0

    rightTokens = new Array(@tokens...)
    leftTokens = []
    leftScreenColumn = 0

    while leftScreenColumn < column
      if leftScreenColumn + rightTokens[0].screenDelta > column
        rightTokens[0..0] = rightTokens[0].splitAt(column - leftScreenColumn)
      nextToken = rightTokens.shift()
      leftScreenColumn += nextToken.screenDelta
      leftTokens.push nextToken

    indentationTokens = @buildSoftWrapIndentationTokens(leftTokens[0], hangingIndent)

    leftFragment = new TokenizedLine(
      tokens: leftTokens
      startBufferColumn: @startBufferColumn
      ruleStack: @ruleStack
      invisibles: @invisibles
      lineEnding: null,
      indentLevel: @indentLevel,
      tabLength: @tabLength
    )
    rightFragment = new TokenizedLine(
      tokens: indentationTokens.concat(rightTokens)
      startBufferColumn: @bufferColumnForScreenColumn(column)
      ruleStack: @ruleStack
      invisibles: @invisibles
      lineEnding: @lineEnding,
      indentLevel: @indentLevel,
      tabLength: @tabLength
    )
    [leftFragment, rightFragment]

  isSoftWrapped: ->
    @lineEnding is null

  isColumnInsideSoftWrapIndentation: (column) ->
    return false if @softWrapIndentationTokens.length is 0

    column < @softWrapIndentationDelta

  getSoftWrapIndentationTokens: ->
    _.select(@tokens, (token) -> token.isSoftWrapIndentation)

  buildSoftWrapIndentationDelta: ->
    _.reduce @softWrapIndentationTokens, ((acc, token) -> acc + token.screenDelta), 0

  hasOnlySoftWrapIndentation: ->
    @tokens.length is @softWrapIndentationTokens.length

  tokenAtBufferColumn: (bufferColumn) ->
    @tokens[@tokenIndexAtBufferColumn(bufferColumn)]

  tokenIndexAtBufferColumn: (bufferColumn) ->
    delta = 0
    for token, index in @tokens
      delta += token.bufferDelta
      return index if delta > bufferColumn
    index - 1

  tokenStartColumnForBufferColumn: (bufferColumn) ->
    delta = 0
    for token in @tokens
      nextDelta = delta + token.bufferDelta
      break if nextDelta > bufferColumn
      delta = nextDelta
    delta

  breakOutAtomicTokens: (inputTokens) ->
    outputTokens = []
    breakOutLeadingSoftTabs = true
    column = @startBufferColumn
    for token in inputTokens
      newTokens = token.breakOutAtomicTokens(@tabLength, breakOutLeadingSoftTabs, column)
      column += newToken.value.length for newToken in newTokens
      outputTokens.push(newTokens...)
      breakOutLeadingSoftTabs = token.isOnlyWhitespace() if breakOutLeadingSoftTabs
    outputTokens

  markLeadingAndTrailingWhitespaceTokens: ->
    @firstNonWhitespaceIndex = @text.search(NonWhitespaceRegex)
    if @firstNonWhitespaceIndex > 0 and isPairedCharacter(@text, @firstNonWhitespaceIndex - 1)
      @firstNonWhitespaceIndex--
    firstTrailingWhitespaceIndex = @text.search(TrailingWhitespaceRegex)
    @lineIsWhitespaceOnly = firstTrailingWhitespaceIndex is 0
    index = 0
    for token in @tokens
      if index < @firstNonWhitespaceIndex
        token.firstNonWhitespaceIndex = Math.min(index + token.value.length, @firstNonWhitespaceIndex - index)
      # Only the *last* segment of a soft-wrapped line can have trailing whitespace
      if @lineEnding? and (index + token.value.length > firstTrailingWhitespaceIndex)
        token.firstTrailingWhitespaceIndex = Math.max(0, firstTrailingWhitespaceIndex - index)
      index += token.value.length
    return

  substituteInvisibleCharacters: ->
    invisibles = @invisibles
    changedText = false

    for token, i in @tokens
      if token.isHardTab
        if invisibles.tab
          token.value = invisibles.tab + token.value.substring(invisibles.tab.length)
          token.hasInvisibleCharacters = true
          changedText = true
      else
        if invisibles.space
          if token.hasLeadingWhitespace() and not token.isSoftWrapIndentation
            token.value = token.value.replace LeadingWhitespaceRegex, (leadingWhitespace) ->
              leadingWhitespace.replace RepeatedSpaceRegex, invisibles.space
            token.hasInvisibleCharacters = true
            changedText = true
          if token.hasTrailingWhitespace()
            token.value = token.value.replace TrailingWhitespaceRegex, (leadingWhitespace) ->
              leadingWhitespace.replace RepeatedSpaceRegex, invisibles.space
            token.hasInvisibleCharacters = true
            changedText = true

    @text = @buildText() if changedText

  buildEndOfLineInvisibles: ->
    @endOfLineInvisibles = []
    {cr, eol} = @invisibles

    switch @lineEnding
      when '\r\n'
        @endOfLineInvisibles.push(cr) if cr
        @endOfLineInvisibles.push(eol) if eol
      when '\n'
        @endOfLineInvisibles.push(eol) if eol

  isComment: ->
    for token in @tokens
      continue if token.scopes.length is 1
      continue if token.isOnlyWhitespace()
      for scope in token.scopes
        return true if _.contains(scope.split('.'), 'comment')
      break
    false

  isOnlyWhitespace: ->
    @lineIsWhitespaceOnly

  tokenAtIndex: (index) ->
    @tokens[index]

  getTokenCount: ->
    @tokens.length

  bufferColumnForToken: (targetToken) ->
    column = 0
    for token in @tokens
      return column if token is targetToken
      column += token.bufferDelta

  getScopeTree: ->
    return @scopeTree if @scopeTree?

    scopeStack = []
    for token in @tokens
      @updateScopeStack(scopeStack, token.scopes)
      _.last(scopeStack).children.push(token)

    @scopeTree = scopeStack[0]
    @updateScopeStack(scopeStack, [])
    @scopeTree

  updateScopeStack: (scopeStack, desiredScopeDescriptor) ->
    # Find a common prefix
    for scope, i in desiredScopeDescriptor
      break unless scopeStack[i]?.scope is desiredScopeDescriptor[i]

    # Pop scopeDescriptor until we're at the common prefx
    until scopeStack.length is i
      poppedScope = scopeStack.pop()
      _.last(scopeStack)?.children.push(poppedScope)

    # Push onto common prefix until scopeStack equals desiredScopeDescriptor
    for j in [i...desiredScopeDescriptor.length]
      scopeStack.push(new Scope(desiredScopeDescriptor[j]))

    return

class Scope
  constructor: (@scope) ->
    @children = []