Convert TokenizedBuffer to JS

src/tokenized-buffer.js

@@ -0,0 +1,586 @@
+const _ = require('underscore-plus')
+const {CompositeDisposable, Emitter} = require('event-kit')
+const {Point, Range} = require('text-buffer')
+const Model = require('./model')
+const TokenizedLine = require('./tokenized-line')
+const TokenIterator = require('./token-iterator')
+const ScopeDescriptor = require('./scope-descriptor')
+const TokenizedBufferIterator = require('./tokenized-buffer-iterator')
+const NullGrammar = require('./null-grammar')
+const {toFirstMateScopeId} = require('./first-mate-helpers')
+
+let nextId = 0
+const prefixedScopes = new Map()
+
+module.exports =
+class TokenizedBuffer {
+  static deserialize (state, atomEnvironment) {
+    const buffer = atomEnvironment.project.bufferForIdSync(state.bufferId)
+    if (!buffer) return null
+
+    state.buffer = buffer
+    state.assert = atomEnvironment.assert
+    return new TokenizedBuffer(state)
+  }
+
+  constructor (params) {
+    this.emitter = new Emitter()
+    this.disposables = new CompositeDisposable()
+    this.tokenIterator = new TokenIterator(this)
+
+    this.alive = true
+    this.id = params.id != null ? params.id : nextId++
+    this.buffer = params.buffer
+    this.tabLength = params.tabLength
+    this.largeFileMode = params.largeFileMode
+    this.assert = params.assert
+
+    this.setGrammar(params.grammar || NullGrammar)
+    this.disposables.add(this.buffer.registerTextDecorationLayer(this))
+  }
+
+  destroy () {
+    if (!this.alive) return
+    this.alive = false
+    this.disposables.dispose()
+    this.tokenizedLines.length = 0
+  }
+
+  isAlive () {
+    return this.alive
+  }
+
+  isDestroyed () {
+    return !this.alive
+  }
+
+  buildIterator () {
+    return new TokenizedBufferIterator(this)
+  }
+
+  classNameForScopeId (id) {
+    const scope = this.grammar.scopeForId(toFirstMateScopeId(id))
+    if (scope) {
+      let prefixedScope = prefixedScopes.get(scope)
+      if (prefixedScope) {
+        return prefixedScope
+      } else {
+        prefixedScope = `syntax--${scope.replace(/\./g, ' syntax--')}`
+        prefixedScopes.set(scope, prefixedScope)
+        return prefixedScope
+      }
+    } else {
+      return null
+    }
+  }
+
+  getInvalidatedRanges () {
+    return []
+  }
+
+  onDidInvalidateRange (fn) {
+    return this.emitter.on('did-invalidate-range', fn)
+  }
+
+  serialize () {
+    return {
+      deserializer: 'TokenizedBuffer',
+      bufferPath: this.buffer.getPath(),
+      bufferId: this.buffer.getId(),
+      tabLength: this.tabLength,
+      largeFileMode: this.largeFileMode
+    }
+  }
+
+  observeGrammar (callback) {
+    callback(this.grammar)
+    return this.onDidChangeGrammar(callback)
+  }
+
+  onDidChangeGrammar (callback) {
+    return this.emitter.on('did-change-grammar', callback)
+  }
+
+  onDidTokenize (callback) {
+    return this.emitter.on('did-tokenize', callback)
+  }
+
+  setGrammar (grammar) {
+    if (!grammar || grammar === this.grammar) return
+
+    this.grammar = grammar
+    this.rootScopeDescriptor = new ScopeDescriptor({scopes: [this.grammar.scopeName]})
+
+    if (this.grammarUpdateDisposable) this.grammarUpdateDisposable.dispose()
+    this.grammarUpdateDisposable = this.grammar.onDidUpdate(() => this.retokenizeLines())
+    this.disposables.add(this.grammarUpdateDisposable)
+
+    this.retokenizeLines()
+    this.emitter.emit('did-change-grammar', grammar)
+  }
+
+  getGrammarSelectionContent () {
+    return this.buffer.getTextInRange([[0, 0], [10, 0]])
+  }
+
+  hasTokenForSelector (selector) {
+    for (const tokenizedLine of this.tokenizedLines) {
+      if (tokenizedLine) {
+        for (let token of tokenizedLine.tokens) {
+          if (selector.matches(token.scopes)) return true
+        }
+      }
+    }
+    return false
+  }
+
+  retokenizeLines () {
+    if (!this.alive) return
+    this.fullyTokenized = false
+    this.tokenizedLines = new Array(this.buffer.getLineCount())
+    this.invalidRows = []
+    if (this.largeFileMode || this.grammar.name === 'Null Grammar') {
+      this.markTokenizationComplete()
+    } else {
+      this.invalidateRow(0)
+    }
+  }
+
+  setVisible (visible) {
+    this.visible = visible
+    if (this.visible && this.grammar.name !== 'Null Grammar' && !this.largeFileMode) {
+      this.tokenizeInBackground()
+    }
+  }
+
+  getTabLength () { return this.tabLength }
+
+  setTabLength (tabLength) {
+    this.tabLength = tabLength
+  }
+
+  tokenizeInBackground () {
+    if (!this.visible || this.pendingChunk || !this.alive) return
+
+    this.pendingChunk = true
+    _.defer(() => {
+      this.pendingChunk = false
+      if (this.isAlive() && this.buffer.isAlive()) this.tokenizeNextChunk()
+    })
+  }
+
+  tokenizeNextChunk () {
+    let rowsRemaining = this.chunkSize
+
+    while (this.firstInvalidRow() != null && rowsRemaining > 0) {
+      var endRow, filledRegion
+      const startRow = this.invalidRows.shift()
+      const lastRow = this.getLastRow()
+      if (startRow > lastRow) continue
+
+      let row = startRow
+      while (true) {
+        const previousStack = this.stackForRow(row)
+        this.tokenizedLines[row] = this.buildTokenizedLineForRow(row, this.stackForRow(row - 1), this.openScopesForRow(row))
+        if (--rowsRemaining === 0) {
+          filledRegion = false
+          endRow = row
+          break
+        }
+        if (row === lastRow || _.isEqual(this.stackForRow(row), previousStack)) {
+          filledRegion = true
+          endRow = row
+          break
+        }
+        row++
+      }
+
+      this.validateRow(endRow)
+      if (!filledRegion) this.invalidateRow(endRow + 1)
+
+      this.emitter.emit('did-invalidate-range', Range(Point(startRow, 0), Point(endRow + 1, 0)))
+    }
+
+    if (this.firstInvalidRow() != null) {
+      this.tokenizeInBackground()
+    } else {
+      this.markTokenizationComplete()
+    }
+  }
+
+  markTokenizationComplete () {
+    if (!this.fullyTokenized) {
+      this.emitter.emit('did-tokenize')
+    }
+    this.fullyTokenized = true
+  }
+
+  firstInvalidRow () {
+    return this.invalidRows[0]
+  }
+
+  validateRow (row) {
+    while (this.invalidRows[0] <= row) this.invalidRows.shift()
+  }
+
+  invalidateRow (row) {
+    this.invalidRows.push(row)
+    this.invalidRows.sort((a, b) => a - b)
+    this.tokenizeInBackground()
+  }
+
+  updateInvalidRows (start, end, delta) {
+    this.invalidRows = this.invalidRows.map((row) => {
+      if (row < start) {
+        return row
+      } else if (start <= row && row <= end) {
+        return end + delta + 1
+      } else if (row > end) {
+        return row + delta
+      }
+    })
+  }
+
+  bufferDidChange (e) {
+    this.changeCount = this.buffer.changeCount
+
+    const {oldRange, newRange} = e
+    const start = oldRange.start.row
+    const end = oldRange.end.row
+    const delta = newRange.end.row - oldRange.end.row
+    const oldLineCount = (oldRange.end.row - oldRange.start.row) + 1
+    const newLineCount = (newRange.end.row - newRange.start.row) + 1
+
+    this.updateInvalidRows(start, end, delta)
+    const previousEndStack = this.stackForRow(end) // used in spill detection below
+    if (this.largeFileMode || (this.grammar.name === 'Null Grammar')) {
+      _.spliceWithArray(this.tokenizedLines, start, oldLineCount, new Array(newLineCount))
+    } else {
+      const newTokenizedLines = this.buildTokenizedLinesForRows(start, end + delta, this.stackForRow(start - 1), this.openScopesForRow(start))
+      _.spliceWithArray(this.tokenizedLines, start, oldLineCount, newTokenizedLines)
+      const newEndStack = this.stackForRow(end + delta)
+      if (newEndStack && !_.isEqual(newEndStack, previousEndStack)) {
+        this.invalidateRow(end + delta + 1)
+      }
+    }
+  }
+
+  isFoldableAtRow (row) {
+    return this.isFoldableCodeAtRow(row) || this.isFoldableCommentAtRow(row)
+  }
+
+  // Returns a {Boolean} indicating whether the given buffer row starts
+  // a a foldable row range due to the code's indentation patterns.
+  isFoldableCodeAtRow (row) {
+    if (row >= 0 && row <= this.buffer.getLastRow()) {
+      const nextRow = this.buffer.nextNonBlankRow(row)
+      const tokenizedLine = this.tokenizedLines[row]
+      if (this.buffer.isRowBlank(row) || (tokenizedLine != null ? tokenizedLine.isComment() : undefined) || (nextRow == null)) {
+        return false
+      } else {
+        return this.indentLevelForRow(nextRow) > this.indentLevelForRow(row)
+      }
+    } else {
+      return false
+    }
+  }
+
+  isFoldableCommentAtRow (row) {
+    const previousRow = row - 1
+    const nextRow = row + 1
+    if (nextRow > this.buffer.getLastRow()) {
+      return false
+    } else {
+      return Boolean(
+        !(this.tokenizedLines[previousRow] != null ? this.tokenizedLines[previousRow].isComment() : undefined) &&
+        (this.tokenizedLines[row] != null ? this.tokenizedLines[row].isComment() : undefined) &&
+        (this.tokenizedLines[nextRow] != null ? this.tokenizedLines[nextRow].isComment() : undefined)
+      )
+    }
+  }
+
+  buildTokenizedLinesForRows (startRow, endRow, startingStack, startingopenScopes) {
+    let ruleStack = startingStack
+    let openScopes = startingopenScopes
+    const stopTokenizingAt = startRow + this.chunkSize
+    const tokenizedLines = []
+    for (let row = startRow, end = endRow; row <= end; row++) {
+      let tokenizedLine
+      if ((ruleStack || (row === 0)) && row < stopTokenizingAt) {
+        tokenizedLine = this.buildTokenizedLineForRow(row, ruleStack, openScopes)
+        ruleStack = tokenizedLine.ruleStack
+        openScopes = this.scopesFromTags(openScopes, tokenizedLine.tags)
+      }
+      tokenizedLines.push(tokenizedLine)
+    }
+
+    if (endRow >= stopTokenizingAt) {
+      this.invalidateRow(stopTokenizingAt)
+      this.tokenizeInBackground()
+    }
+
+    return tokenizedLines
+  }
+
+  buildTokenizedLineForRow (row, ruleStack, openScopes) {
+    return this.buildTokenizedLineForRowWithText(row, this.buffer.lineForRow(row), ruleStack, openScopes)
+  }
+
+  buildTokenizedLineForRowWithText (row, text, currentRuleStack = this.stackForRow(row - 1), openScopes = this.openScopesForRow(row)) {
+    const lineEnding = this.buffer.lineEndingForRow(row);
+    const {tags, ruleStack} = this.grammar.tokenizeLine(text, currentRuleStack, row === 0, false)
+    return new TokenizedLine({
+      openScopes,
+      text,
+      tags,
+      ruleStack,
+      lineEnding,
+      tokenIterator: this.tokenIterator,
+      grammar: this.grammar
+    })
+  }
+
+  tokenizedLineForRow (bufferRow) {
+    if (bufferRow >= 0 && bufferRow <= this.buffer.getLastRow()) {
+      const tokenizedLine = this.tokenizedLines[bufferRow]
+      if (tokenizedLine) {
+        return tokenizedLine
+      } else {
+        const text = this.buffer.lineForRow(bufferRow)
+        const lineEnding = this.buffer.lineEndingForRow(bufferRow)
+        const tags = [
+          this.grammar.startIdForScope(this.grammar.scopeName),
+          text.length,
+          this.grammar.endIdForScope(this.grammar.scopeName)
+        ]
+        return this.tokenizedLines[bufferRow] = new TokenizedLine({
+          openScopes: [],
+          text,
+          tags,
+          lineEnding,
+          tokenIterator: this.tokenIterator,
+          grammar: this.grammar
+        })
+      }
+    }
+  }
+
+  tokenizedLinesForRows (startRow, endRow) {
+    const result = []
+    for (let row = startRow, end = endRow; row <= end; row++) {
+      result.push(this.tokenizedLineForRow(row))
+    }
+    return result
+  }
+
+  stackForRow (bufferRow) {
+    return this.tokenizedLines[bufferRow] && this.tokenizedLines[bufferRow].ruleStack
+  }
+
+  openScopesForRow (bufferRow) {
+    const precedingLine = this.tokenizedLines[bufferRow - 1]
+    if (precedingLine) {
+      return this.scopesFromTags(precedingLine.openScopes, precedingLine.tags)
+    } else {
+      return []
+    }
+  }
+
+  scopesFromTags (startingScopes, tags) {
+    const scopes = startingScopes.slice()
+    for (const tag of tags) {
+      if (tag < 0) {
+        if (tag % 2 === -1) {
+          scopes.push(tag)
+        } else {
+          const matchingStartTag = tag + 1
+          while (true) {
+            if (scopes.pop() === matchingStartTag) break
+            if (scopes.length === 0) {
+              this.assert(false, 'Encountered an unmatched scope end tag.', error => {
+                error.metadata = {
+                  grammarScopeName: this.grammar.scopeName,
+                  unmatchedEndTag: this.grammar.scopeForId(tag)
+                }
+                const path = require('path')
+                error.privateMetadataDescription = `The contents of \`${path.basename(this.buffer.getPath())}\``
+                return error.privateMetadata = {
+                  filePath: this.buffer.getPath(),
+                  fileContents: this.buffer.getText()
+                }
+              })
+              break
+            }
+          }
+        }
+      }
+    }
+    return scopes
+  }
+
+  indentLevelForRow (bufferRow) {
+    const line = this.buffer.lineForRow(bufferRow)
+    let indentLevel = 0
+
+    if (line === '') {
+      let nextRow = bufferRow + 1
+      const lineCount = this.getLineCount()
+      while (nextRow < lineCount) {
+        const nextLine = this.buffer.lineForRow(nextRow)
+        if (nextLine !== '') {
+          indentLevel = Math.ceil(this.indentLevelForLine(nextLine))
+          break
+        }
+        nextRow++
+      }
+
+      let previousRow = bufferRow - 1
+      while (previousRow >= 0) {
+        const previousLine = this.buffer.lineForRow(previousRow)
+        if (previousLine !== '') {
+          indentLevel = Math.max(Math.ceil(this.indentLevelForLine(previousLine)), indentLevel)
+          break
+        }
+        previousRow--
+      }
+
+      return indentLevel
+    } else {
+      return this.indentLevelForLine(line)
+    }
+  }
+
+  indentLevelForLine (line, tabLength = this.tabLength) {
+    let indentLength = 0
+    for (let i = 0, {length} = line; i < length; i++) {
+      const char = line[i]
+      if (char === '\t') {
+        indentLength += tabLength - (indentLength % tabLength)
+      } else if (char === ' ') {
+        indentLength++
+      } else {
+        break
+      }
+    }
+    return indentLength / tabLength
+  }
+
+  scopeDescriptorForPosition (position) {
+    let scopes
+    const {row, column} = this.buffer.clipPosition(Point.fromObject(position))
+
+    const iterator = this.tokenizedLineForRow(row).getTokenIterator()
+    while (iterator.next()) {
+      if (iterator.getBufferEnd() > column) {
+        scopes = iterator.getScopes()
+        break
+      }
+    }
+
+    // rebuild scope of last token if we iterated off the end
+    if (!scopes) {
+      scopes = iterator.getScopes()
+      scopes.push(...iterator.getScopeEnds().reverse())
+    }
+
+    return new ScopeDescriptor({scopes})
+  }
+
+  tokenForPosition (position) {
+    const {row, column} = Point.fromObject(position)
+    return this.tokenizedLineForRow(row).tokenAtBufferColumn(column)
+  }
+
+  tokenStartPositionForPosition (position) {
+    let {row, column} = Point.fromObject(position)
+    column = this.tokenizedLineForRow(row).tokenStartColumnForBufferColumn(column)
+    return new Point(row, column)
+  }
+
+  bufferRangeForScopeAtPosition (selector, position) {
+    let endColumn, tag, tokenIndex
+    position = Point.fromObject(position)
+
+    const {openScopes, tags} = this.tokenizedLineForRow(position.row)
+    const scopes = openScopes.map(tag => this.grammar.scopeForId(tag))
+
+    let startColumn = 0
+    for (tokenIndex = 0; tokenIndex < tags.length; tokenIndex++) {
+      tag = tags[tokenIndex]
+      if (tag < 0) {
+        if ((tag % 2) === -1) {
+          scopes.push(this.grammar.scopeForId(tag))
+        } else {
+          scopes.pop()
+        }
+      } else {
+        endColumn = startColumn + tag
+        if (endColumn >= position.column) {
+          break
+        } else {
+          startColumn = endColumn
+        }
+      }
+    }
+
+    if (!selectorMatchesAnyScope(selector, scopes)) return
+
+    const startScopes = scopes.slice()
+    for (let startTokenIndex = tokenIndex - 1; startTokenIndex >= 0; startTokenIndex--) {
+      tag = tags[startTokenIndex]
+      if (tag < 0) {
+        if ((tag % 2) === -1) {
+          startScopes.pop()
+        } else {
+          startScopes.push(this.grammar.scopeForId(tag))
+        }
+      } else {
+        if (!selectorMatchesAnyScope(selector, startScopes)) { break }
+        startColumn -= tag
+      }
+    }
+
+    const endScopes = scopes.slice()
+    for (let endTokenIndex = tokenIndex + 1, end = tags.length; endTokenIndex < end; endTokenIndex++) {
+      tag = tags[endTokenIndex]
+      if (tag < 0) {
+        if ((tag % 2) === -1) {
+          endScopes.push(this.grammar.scopeForId(tag))
+        } else {
+          endScopes.pop()
+        }
+      } else {
+        if (!selectorMatchesAnyScope(selector, endScopes)) { break }
+        endColumn += tag
+      }
+    }
+
+    return new Range(new Point(position.row, startColumn), new Point(position.row, endColumn))
+  }
+
+  // Gets the row number of the last line.
+  //
+  // Returns a {Number}.
+  getLastRow () {
+    return this.buffer.getLastRow()
+  }
+
+  getLineCount () {
+    return this.buffer.getLineCount()
+  }
+
+  logLines (start = 0, end = this.buffer.getLastRow()) {
+    for (let row = start; row <= end1; row++) {
+      const line = this.tokenizedLines[row].text
+      console.log(row, line, line.length)
+    }
+  }
+}
+
+function selectorMatchesAnyScope (selector, scopes) {
+  const targetClasses = selector.replace(/^\./, '').split('.')
+  return scopes.some((scope) => {
+    const scopeClasses = scope.split('.')
+    return _.isSubset(targetClasses, scopeClasses)
+  })
+}