Start work on switching to new language mode API

src/grammar-registry.js

@@ -1,9 +1,12 @@
 const _ = require('underscore-plus')
 const FirstMate = require('first-mate')
+const {Disposable, CompositeDisposable} = require('event-kit')
+const TokenizedBuffer = require('./tokenized-buffer')
 const Token = require('./token')
 const fs = require('fs-plus')
-const Grim = require('grim')
+const {Point, Range} = require('text-buffer')
 
+const GRAMMAR_SELECTION_RANGE = Range(Point.ZERO, Point(10, 0)).freeze()
 const PathSplitRegex = new RegExp('[/.]')
 
 // Extended: Syntax class holding the grammars used for tokenizing.
@@ -17,12 +20,59 @@ class GrammarRegistry extends FirstMate.GrammarRegistry {
   constructor ({config} = {}) {
     super({maxTokensPerLine: 100, maxLineLength: 1000})
     this.config = config
+    this.grammarOverridesByPath = {}
+    this.grammarScoresByBuffer = new Map()
+    this.subscriptions = new CompositeDisposable()
+
+    const grammarAddedOrUpdated = this.grammarAddedOrUpdated.bind(this)
+    this.onDidAddGrammar(grammarAddedOrUpdated)
+    this.onDidUpdateGrammar(grammarAddedOrUpdated)
   }
 
   createToken (value, scopes) {
     return new Token({value, scopes})
   }
 
+  maintainGrammar (buffer) {
+    this.assignLanguageModeToBuffer(buffer)
+
+    const pathChangeSubscription = buffer.onDidChangePath(() => {
+      this.grammarScoresByBuffer.delete(buffer)
+      this.assignLanguageModeToBuffer(buffer)
+    })
+
+    this.subscriptions.add(pathChangeSubscription)
+
+    return new Disposable(() => {
+      this.subscriptions.remove(pathChangeSubscription)
+      pathChangeSubscription.dispose()
+    })
+  }
+
+  assignLanguageModeToBuffer (buffer) {
+    let grammar = null
+    const overrideScopeName = this.grammarOverridesByPath[buffer.getPath()]
+    if (overrideScopeName) {
+      grammar = this.grammarForScopeName(overrideScopeName)
+      this.grammarScoresByBuffer.set(buffer, null)
+    }
+
+    if (!grammar) {
+      const result = this.selectGrammarWithScore(
+        buffer.getPath(),
+        buffer.getTextInRange(GRAMMAR_SELECTION_RANGE)
+      )
+      grammar = result.grammar
+      this.grammarScoresByBuffer.set(buffer, result.score)
+    }
+
+    buffer.setLanguageMode(this.languageModeForGrammarAndBuffer(grammar, buffer))
+  }
+
+  languageModeForGrammarAndBuffer (grammar, buffer) {
+    return new TokenizedBuffer({grammar, buffer, config: this.config})
+  }
+
   // Extended: Select a grammar for the given file path and file contents.
   //
   // This picks the best match by checking the file path and contents against
@@ -120,52 +170,62 @@ class GrammarRegistry extends FirstMate.GrammarRegistry {
     return grammar.firstLineRegex.testSync(lines.slice(0, numberOfNewlinesInRegex + 1).join('\n'))
   }
 
-  // Deprecated: Get the grammar override for the given file path.
+  // Get the grammar override for the given file path.
   //
   // * `filePath` A {String} file path.
   //
   // Returns a {String} such as `"source.js"`.
   grammarOverrideForPath (filePath) {
-    Grim.deprecate('Use atom.textEditors.getGrammarOverride(editor) instead')
-
-    const editor = getEditorForPath(filePath)
-    if (editor) {
-      return atom.textEditors.getGrammarOverride(editor)
-    }
+    return this.grammarOverridesByPath[filePath]
   }
 
-  // Deprecated: Set the grammar override for the given file path.
+  // Set the grammar override for the given file path.
   //
   // * `filePath` A non-empty {String} file path.
   // * `scopeName` A {String} such as `"source.js"`.
   //
   // Returns undefined.
   setGrammarOverrideForPath (filePath, scopeName) {
-    Grim.deprecate('Use atom.textEditors.setGrammarOverride(editor, scopeName) instead')
-
-    const editor = getEditorForPath(filePath)
-    if (editor) {
-      atom.textEditors.setGrammarOverride(editor, scopeName)
-    }
+    this.grammarOverridesByPath[filePath] = scopeName
   }
 
-  // Deprecated: Remove the grammar override for the given file path.
+  // Remove the grammar override for the given file path.
   //
   // * `filePath` A {String} file path.
   //
   // Returns undefined.
   clearGrammarOverrideForPath (filePath) {
-    Grim.deprecate('Use atom.textEditors.clearGrammarOverride(editor) instead')
+    delete this.grammarOverridesByPath[filePath]
+  }
 
-    const editor = getEditorForPath(filePath)
-    if (editor) {
-      atom.textEditors.clearGrammarOverride(editor)
-    }
-  }
-}
-
-function getEditorForPath (filePath) {
-  if (filePath != null) {
-    return atom.workspace.getTextEditors().find(editor => editor.getPath() === filePath)
+  grammarAddedOrUpdated (grammar) {
+    this.grammarScoresByBuffer.forEach((score, buffer) => {
+      const languageMode = buffer.getLanguageMode()
+      if (grammar.injectionSelector) {
+        if (languageMode.hasTokenForSelector(grammar.injectionSelector)) {
+          languageMode.retokenizeLines()
+        }
+        return
+      }
+
+      const grammarOverride = this.grammarOverridesByPath[buffer.getPath()]
+      if (grammarOverride) {
+        if (grammar.scopeName === grammarOverride) {
+          buffer.setLanguageMode(this.languageModeForGrammarAndBuffer(grammar, buffer))
+        }
+      } else {
+        const score = this.getGrammarScore(
+          grammar,
+          buffer.getPath(),
+          buffer.getTextInRange(GRAMMAR_SELECTION_RANGE)
+        )
+
+        let currentScore = this.grammarScoresByBuffer.get(buffer)
+        if (currentScore == null || score > currentScore) {
+          buffer.setLanguageMode(this.languageModeForGrammarAndBuffer(grammar, buffer))
+          this.grammarScoresByBuffer.set(buffer, score)
+        }
+      }
+    })
   }
 }

src/project.js

@@ -24,6 +24,7 @@ class Project extends Model {
     this.notificationManager = notificationManager
     this.applicationDelegate = applicationDelegate
     this.grammarRegistry = grammarRegistry
+
     this.emitter = new Emitter()
     this.buffers = []
     this.rootDirectories = []
@@ -105,6 +106,7 @@ class Project extends Model {
     return Promise.all(bufferPromises).then(buffers => {
       this.buffers = buffers.filter(Boolean)
       for (let buffer of this.buffers) {
+        this.grammarRegistry.maintainGrammar(buffer)
         this.subscribeToBuffer(buffer)
       }
       this.setPaths(state.paths || [], {mustExist: true, exact: true})
@@ -648,6 +650,7 @@ class Project extends Model {
 
   addBuffer (buffer, options = {}) {
     this.buffers.push(buffer)
+    this.grammarRegistry.maintainGrammar(buffer)
     this.subscribeToBuffer(buffer)
     this.emitter.emit('did-add-buffer', buffer)
     return buffer

src/text-editor-registry.js

@@ -1,9 +1,9 @@
 /** @babel */
 
 import {Emitter, Disposable, CompositeDisposable} from 'event-kit'
-import {Point, Range} from 'text-buffer'
 import TextEditor from './text-editor'
 import ScopeDescriptor from './scope-descriptor'
+import Grim from 'grim'
 
 const EDITOR_PARAMS_BY_SETTING_KEY = [
   ['core.fileEncoding', 'encoding'],
@@ -27,8 +27,6 @@ const EDITOR_PARAMS_BY_SETTING_KEY = [
   ['editor.scrollSensitivity', 'scrollSensitivity']
 ]
 
-const GRAMMAR_SELECTION_RANGE = Range(Point.ZERO, Point(10, 0)).freeze()
-
 // Experimental: This global registry tracks registered `TextEditors`.
 //
 // If you want to add functionality to a wider set of text editors than just
@@ -41,12 +39,9 @@ const GRAMMAR_SELECTION_RANGE = Range(Point.ZERO, Point(10, 0)).freeze()
 // done using your editor, be sure to call `dispose` on the returned disposable
 // to avoid leaking editors.
 export default class TextEditorRegistry {
-  constructor ({config, grammarRegistry, assert, packageManager}) {
+  constructor ({config, assert, packageManager}) {
     this.assert = assert
     this.config = config
-    this.grammarRegistry = grammarRegistry
-    this.scopedSettingsDelegate = new ScopedSettingsDelegate(config)
-    this.grammarAddedOrUpdated = this.grammarAddedOrUpdated.bind(this)
     this.clear()
 
     this.initialPackageActivationPromise = new Promise((resolve) => {
@@ -83,10 +78,6 @@ export default class TextEditorRegistry {
     this.editorsWithMaintainedGrammar = new Set()
     this.editorGrammarOverrides = {}
     this.editorGrammarScores = new WeakMap()
-    this.subscriptions.add(
-      this.grammarRegistry.onDidAddGrammar(this.grammarAddedOrUpdated),
-      this.grammarRegistry.onDidUpdateGrammar(this.grammarAddedOrUpdated)
-    )
   }
 
   destroy () {
@@ -114,10 +105,7 @@ export default class TextEditorRegistry {
 
     let scope = null
     if (params.buffer) {
-      const filePath = params.buffer.getPath()
-      const headContent = params.buffer.getTextInRange(GRAMMAR_SELECTION_RANGE)
-      params.grammar = this.grammarRegistry.selectGrammar(filePath, headContent)
-      scope = new ScopeDescriptor({scopes: [params.grammar.scopeName]})
+      scope = new ScopeDescriptor({scopes: [params.buffer.getLanguageMode().getGrammar().scopeName]})
     }
 
     Object.assign(params, this.textEditorParamsForScope(scope))
@@ -159,8 +147,6 @@ export default class TextEditorRegistry {
     }
     this.editorsWithMaintainedConfig.add(editor)
 
-    editor.setScopedSettingsDelegate(this.scopedSettingsDelegate)
-
     this.subscribeToSettingsForEditorScope(editor)
     const grammarChangeSubscription = editor.onDidChangeGrammar(() => {
       this.subscribeToSettingsForEditorScope(editor)
@@ -182,7 +168,6 @@ export default class TextEditorRegistry {
 
     return new Disposable(() => {
       this.editorsWithMaintainedConfig.delete(editor)
-      editor.setScopedSettingsDelegate(null)
       tokenizeSubscription.dispose()
       grammarChangeSubscription.dispose()
       this.subscriptions.remove(grammarChangeSubscription)
@@ -190,134 +175,47 @@ export default class TextEditorRegistry {
     })
   }
 
-  // Set a {TextEditor}'s grammar based on its path and content, and continue
-  // to update its grammar as grammars are added or updated, or the editor's
-  // file path changes.
+  // Deprecated: set a {TextEditor}'s grammar based on its path and content,
+  // and continue to update its grammar as grammars are added or updated, or
+  // the editor's file path changes.
   //
   // * `editor` The editor whose grammar will be maintained.
   //
   // Returns a {Disposable} that can be used to stop updating the editor's
   // grammar.
   maintainGrammar (editor) {
-    if (this.editorsWithMaintainedGrammar.has(editor)) {
-      return new Disposable(noop)
-    }
-
-    this.editorsWithMaintainedGrammar.add(editor)
-
-    const buffer = editor.getBuffer()
-    for (let existingEditor of this.editorsWithMaintainedGrammar) {
-      if (existingEditor.getBuffer() === buffer) {
-        const existingOverride = this.editorGrammarOverrides[existingEditor.id]
-        if (existingOverride) {
-          this.editorGrammarOverrides[editor.id] = existingOverride
-        }
-        break
-      }
-    }
-
-    this.selectGrammarForEditor(editor)
-
-    const pathChangeSubscription = editor.onDidChangePath(() => {
-      this.editorGrammarScores.delete(editor)
-      this.selectGrammarForEditor(editor)
-    })
-
-    this.subscriptions.add(pathChangeSubscription)
-
-    return new Disposable(() => {
-      delete this.editorGrammarOverrides[editor.id]
-      this.editorsWithMaintainedGrammar.delete(editor)
-      this.subscriptions.remove(pathChangeSubscription)
-      pathChangeSubscription.dispose()
-    })
+    Grim.deprecate('Use atom.grammars.maintainGrammar(buffer) instead.')
+    atom.grammars.maintainGrammar(editor.getBuffer())
   }
 
-  // Force a {TextEditor} to use a different grammar than the one that would
-  // otherwise be selected for it.
+  // Deprecated: Force a {TextEditor} to use a different grammar than the
+  // one that would otherwise be selected for it.
   //
   // * `editor` The editor whose gramamr will be set.
   // * `scopeName` The {String} root scope name for the desired {Grammar}.
   setGrammarOverride (editor, scopeName) {
-    this.editorGrammarOverrides[editor.id] = scopeName
-    this.editorGrammarScores.delete(editor)
-    editor.setGrammar(this.grammarRegistry.grammarForScopeName(scopeName))
+    Grim.deprecate('Use atom.grammars.setGrammarOverrideForPath(filePath) instead.')
+    atom.grammars.setGrammarOverrideForPath(editor.getPath(), scopeName)
   }
 
-  // Retrieve the grammar scope name that has been set as a grammar override
-  // for the given {TextEditor}.
+  // Deprecated: Retrieve the grammar scope name that has been set as a
+  // grammar override for the given {TextEditor}.
   //
   // * `editor` The editor.
   //
   // Returns a {String} scope name, or `null` if no override has been set
   // for the given editor.
   getGrammarOverride (editor) {
-    return this.editorGrammarOverrides[editor.id]
+    Grim.deprecate('Use atom.grammars.grammarOverrideForPath(filePath) instead.')
+    return atom.grammars.grammarOverrideForPath(editor.getPath())
   }
 
-  // Remove any grammar override that has been set for the given {TextEditor}.
+  // Deprecated: Remove any grammar override that has been set for the given {TextEditor}.
   //
   // * `editor` The editor.
   clearGrammarOverride (editor) {
-    delete this.editorGrammarOverrides[editor.id]
-    this.selectGrammarForEditor(editor)
-  }
-
-  // Private
-
-  grammarAddedOrUpdated (grammar) {
-    this.editorsWithMaintainedGrammar.forEach((editor) => {
-      if (grammar.injectionSelector) {
-        if (editor.tokenizedBuffer.hasTokenForSelector(grammar.injectionSelector)) {
-          editor.tokenizedBuffer.retokenizeLines()
-        }
-        return
-      }
-
-      const grammarOverride = this.editorGrammarOverrides[editor.id]
-      if (grammarOverride) {
-        if (grammar.scopeName === grammarOverride) {
-          editor.setGrammar(grammar)
-        }
-      } else {
-        const score = this.grammarRegistry.getGrammarScore(
-          grammar,
-          editor.getPath(),
-          editor.getTextInBufferRange(GRAMMAR_SELECTION_RANGE)
-        )
-
-        let currentScore = this.editorGrammarScores.get(editor)
-        if (currentScore == null || score > currentScore) {
-          editor.setGrammar(grammar)
-          this.editorGrammarScores.set(editor, score)
-        }
-      }
-    })
-  }
-
-  selectGrammarForEditor (editor) {
-    const grammarOverride = this.editorGrammarOverrides[editor.id]
-
-    if (grammarOverride) {
-      const grammar = this.grammarRegistry.grammarForScopeName(grammarOverride)
-      editor.setGrammar(grammar)
-      return
-    }
-
-    const {grammar, score} = this.grammarRegistry.selectGrammarWithScore(
-      editor.getPath(),
-      editor.getTextInBufferRange(GRAMMAR_SELECTION_RANGE)
-    )
-
-    if (!grammar) {
-      throw new Error(`No grammar found for path: ${editor.getPath()}`)
-    }
-
-    const currentScore = this.editorGrammarScores.get(editor)
-    if (currentScore == null || score > currentScore) {
-      editor.setGrammar(grammar)
-      this.editorGrammarScores.set(editor, score)
-    }
+    Grim.deprecate('Use atom.grammars.clearGrammarOverrideForPath(filePath) instead.')
+    atom.grammars.clearGrammarOverrideForPath(editor.getPath())
   }
 
   async subscribeToSettingsForEditorScope (editor) {
@@ -390,44 +288,3 @@ function shouldEditorUseSoftTabs (editor, tabType, softTabs) {
 }
 
 function noop () {}
-
-class ScopedSettingsDelegate {
-  constructor (config) {
-    this.config = config
-  }
-
-  getNonWordCharacters (scope) {
-    return this.config.get('editor.nonWordCharacters', {scope: scope})
-  }
-
-  getIncreaseIndentPattern (scope) {
-    return this.config.get('editor.increaseIndentPattern', {scope: scope})
-  }
-
-  getDecreaseIndentPattern (scope) {
-    return this.config.get('editor.decreaseIndentPattern', {scope: scope})
-  }
-
-  getDecreaseNextIndentPattern (scope) {
-    return this.config.get('editor.decreaseNextIndentPattern', {scope: scope})
-  }
-
-  getFoldEndPattern (scope) {
-    return this.config.get('editor.foldEndPattern', {scope: scope})
-  }
-
-  getCommentStrings (scope) {
-    const commentStartEntries = this.config.getAll('editor.commentStart', {scope})
-    const commentEndEntries = this.config.getAll('editor.commentEnd', {scope})
-    const commentStartEntry = commentStartEntries[0]
-    const commentEndEntry = commentEndEntries.find((entry) => {
-      return entry.scopeSelector === commentStartEntry.scopeSelector
-    })
-    return {
-      commentStartString: commentStartEntry && commentStartEntry.value,
-      commentEndString: commentEndEntry && commentEndEntry.value
-    }
-  }
-}
-
-TextEditorRegistry.ScopedSettingsDelegate = ScopedSettingsDelegate

src/text-editor.js

@@ -7,9 +7,9 @@ const {CompositeDisposable, Disposable, Emitter} = require('event-kit')
 const TextBuffer = require('text-buffer')
 const {Point, Range} = TextBuffer
 const DecorationManager = require('./decoration-manager')
-const TokenizedBuffer = require('./tokenized-buffer')
 const Cursor = require('./cursor')
 const Selection = require('./selection')
+const NullGrammar = require('./null-grammar')
 
 const TextMateScopeSelector = require('first-mate').ScopeSelector
 const GutterContainer = require('./gutter-container')
@@ -86,12 +86,13 @@ class TextEditor {
   static deserialize (state, atomEnvironment) {
     if (state.version !== SERIALIZATION_VERSION) return null
 
-    try {
-      const tokenizedBuffer = TokenizedBuffer.deserialize(state.tokenizedBuffer, atomEnvironment)
-      if (!tokenizedBuffer) return null
+    let bufferId = state.tokenizedBuffer
+      ? state.tokenizedBuffer.bufferId
+      : state.bufferId
 
-      state.tokenizedBuffer = tokenizedBuffer
-      state.tabLength = state.tokenizedBuffer.getTabLength()
+    try {
+      state.buffer = atomEnvironment.project.bufferForIdSync(bufferId)
+      if (!state.buffer) return null
     } catch (error) {
       if (error.syscall === 'read') {
         return // Error reading the file, don't deserialize an editor for it
@@ -100,7 +101,6 @@ class TextEditor {
       }
     }
 
-    state.buffer = state.tokenizedBuffer.buffer
     state.assert = atomEnvironment.assert.bind(atomEnvironment)
     const editor = new TextEditor(state)
     if (state.registered) {
@@ -175,13 +175,8 @@ class TextEditor {
       shouldDestroyOnFileDelete () { return atom.config.get('core.closeDeletedFileTabs') }
     })
 
-    this.tokenizedBuffer = params.tokenizedBuffer || new TokenizedBuffer({
-      grammar: params.grammar,
-      tabLength,
-      buffer: this.buffer,
-      largeFileMode: this.largeFileMode,
-      assert: this.assert
-    })
+    const languageMode = this.buffer.getLanguageMode()
+    if (languageMode && languageMode.setTabLength) languageMode.setTabLength(tabLength)
 
     if (params.displayLayer) {
       this.displayLayer = params.displayLayer
@@ -217,8 +212,6 @@ class TextEditor {
       this.selectionsMarkerLayer = this.addMarkerLayer({maintainHistory: true, persistent: true})
     }
 
-    this.displayLayer.setTextDecorationLayer(this.tokenizedBuffer)
-
     this.decorationManager = new DecorationManager(this)
     this.decorateMarkerLayer(this.selectionsMarkerLayer, {type: 'cursor'})
     if (!this.isMini()) this.decorateCursorLine()
@@ -271,9 +264,8 @@ class TextEditor {
     return this
   }
 
-  get languageMode () {
-    return this.tokenizedBuffer
-  }
+  get languageMode () { return this.buffer.getLanguageMode() }
+  get tokenizedBuffer () { return this.buffer.getLanguageMode() }
 
   get rowsPerPage () {
     return this.getRowsPerPage()
@@ -344,8 +336,8 @@ class TextEditor {
           break
 
         case 'tabLength':
-          if (value > 0 && value !== this.tokenizedBuffer.getTabLength()) {
-            this.tokenizedBuffer.setTabLength(value)
+          if (value > 0 && value !== this.displayLayer.tabLength) {
+            this.buffer.getLanguageMode().setTabLength(value)
             displayLayerParams.tabLength = value
           }
           break
@@ -513,16 +505,10 @@ class TextEditor {
   }
 
   serialize () {
-    const tokenizedBufferState = this.tokenizedBuffer.serialize()
-
     return {
       deserializer: 'TextEditor',
       version: SERIALIZATION_VERSION,
 
-      // TODO: Remove this forward-compatible fallback once 1.8 reaches stable.
-      displayBuffer: {tokenizedBuffer: tokenizedBufferState},
-
-      tokenizedBuffer: tokenizedBufferState,
       displayLayerId: this.displayLayer.id,
       selectionsMarkerLayerId: this.selectionsMarkerLayer.id,
 
@@ -533,6 +519,7 @@ class TextEditor {
       softWrapHangingIndentLength: this.displayLayer.softWrapHangingIndent,
 
       id: this.id,
+      bufferId: this.buffer.id,
       softTabs: this.softTabs,
       softWrapped: this.softWrapped,
       softWrapAtPreferredLineLength: this.softWrapAtPreferredLineLength,
@@ -553,6 +540,7 @@ class TextEditor {
 
   subscribeToBuffer () {
     this.buffer.retain()
+    this.disposables.add(this.buffer.onDidChangeLanguageMode(this.handleLanguageModeChange.bind(this)))
     this.disposables.add(this.buffer.onDidChangePath(() => {
       this.emitter.emit('did-change-title', this.getTitle())
       this.emitter.emit('did-change-path', this.getPath())
@@ -576,7 +564,6 @@ class TextEditor {
   }
 
   subscribeToDisplayLayer () {
-    this.disposables.add(this.tokenizedBuffer.onDidChangeGrammar(this.handleGrammarChange.bind(this)))
     this.disposables.add(this.displayLayer.onDidChange(changes => {
       this.mergeIntersectingSelections()
       if (this.component) this.component.didChangeDisplayLayer(changes)
@@ -596,7 +583,6 @@ class TextEditor {
     this.alive = false
     this.disposables.dispose()
     this.displayLayer.destroy()
-    this.tokenizedBuffer.destroy()
     for (let selection of this.selections.slice()) {
       selection.destroy()
     }
@@ -731,7 +717,9 @@ class TextEditor {
   //
   // Returns a {Disposable} on which `.dispose()` can be called to unsubscribe.
   onDidChangeGrammar (callback) {
-    return this.emitter.on('did-change-grammar', callback)
+    return this.buffer.onDidChangeLanguageMode(() => {
+      callback(this.buffer.getLanguageMode().getGrammar())
+    })
   }
 
   // Extended: Calls your `callback` when the result of {::isModified} changes.
@@ -947,7 +935,7 @@ class TextEditor {
       selectionsMarkerLayer,
       softTabs,
       suppressCursorCreation: true,
-      tabLength: this.tokenizedBuffer.getTabLength(),
+      tabLength: this.getTabLength(),
       initialScrollTopRow: this.getScrollTopRow(),
       initialScrollLeftColumn: this.getScrollLeftColumn(),
       assert: this.assert,
@@ -960,7 +948,10 @@ class TextEditor {
   }
 
   // Controls visibility based on the given {Boolean}.
-  setVisible (visible) { this.tokenizedBuffer.setVisible(visible) }
+  setVisible (visible) {
+    const languageMode = this.buffer.getLanguageMode()
+    languageMode.setVisible(visible)
+  }
 
   setMini (mini) {
     this.update({mini})
@@ -3353,7 +3344,7 @@ class TextEditor {
   // Essential: Get the on-screen length of tab characters.
   //
   // Returns a {Number}.
-  getTabLength () { return this.tokenizedBuffer.getTabLength() }
+  getTabLength () { return this.displayLayer.tabLength }
 
   // Essential: Set the on-screen length of tab characters. Setting this to a
   // {Number} This will override the `editor.tabLength` setting.
@@ -3384,9 +3375,9 @@ class TextEditor {
   // Returns a {Boolean} or undefined if no non-comment lines had leading
   // whitespace.
   usesSoftTabs () {
+    const languageMode = this.buffer.getLanguageMode()
     for (let bufferRow = 0, end = Math.min(1000, this.buffer.getLastRow()); bufferRow <= end; bufferRow++) {
-      const tokenizedLine = this.tokenizedBuffer.tokenizedLines[bufferRow]
-      if (tokenizedLine && tokenizedLine.isComment()) continue
+      if (languageMode.isRowCommented(bufferRow)) continue
       const line = this.buffer.lineForRow(bufferRow)
       if (line[0] === ' ') return true
       if (line[0] === '\t') return false
@@ -3509,7 +3500,19 @@ class TextEditor {
   //
   // Returns a {Number}.
   indentLevelForLine (line) {
-    return this.tokenizedBuffer.indentLevelForLine(line)
+    const tabLength = this.getTabLength()
+    let indentLength = 0
+    for (let i = 0, {length} = line; i < length; i++) {
+      const char = line[i]
+      if (char === '\t') {
+        indentLength += tabLength - (indentLength % tabLength)
+      } else if (char === ' ') {
+        indentLength++
+      } else {
+        break
+      }
+    }
+    return indentLength / tabLength
   }
 
   // Extended: Indent rows intersecting selections based on the grammar's suggested
@@ -3542,7 +3545,7 @@ class TextEditor {
 
   // Essential: Get the current {Grammar} of this editor.
   getGrammar () {
-    return this.tokenizedBuffer.grammar
+    return this.buffer.getLanguageMode().getGrammar() || NullGrammar
   }
 
   // Essential: Set the current {Grammar} of this editor.
@@ -3552,17 +3555,13 @@ class TextEditor {
   //
   // * `grammar` {Grammar}
   setGrammar (grammar) {
-    return this.tokenizedBuffer.setGrammar(grammar)
-  }
-
-  // Reload the grammar based on the file name.
-  reloadGrammar () {
-    return this.tokenizedBuffer.reloadGrammar()
+    Grim.deprecate('Use atom.grammars.setGrammarOverrideForPath(filePath) instead')
+    atom.grammars.setGrammarOverrideForPath(this.getPath(), grammar.scopeName)
   }
 
   // Experimental: Get a notification when async tokenization is completed.
   onDidTokenize (callback) {
-    return this.tokenizedBuffer.onDidTokenize(callback)
+    return this.buffer.getLanguageMode().onDidTokenize(callback)
   }
 
   /*
@@ -3573,7 +3572,7 @@ class TextEditor {
   // e.g. `['.source.ruby']`, or `['.source.coffee']`. You can use this with
   // {Config::get} to get language specific config values.
   getRootScopeDescriptor () {
-    return this.tokenizedBuffer.rootScopeDescriptor
+    return this.buffer.getLanguageMode().rootScopeDescriptor
   }
 
   // Essential: Get the syntactic scopeDescriptor for the given position in buffer
@@ -3587,7 +3586,7 @@ class TextEditor {
   //
   // Returns a {ScopeDescriptor}.
   scopeDescriptorForBufferPosition (bufferPosition) {
-    return this.tokenizedBuffer.scopeDescriptorForPosition(bufferPosition)
+    return this.buffer.getLanguageMode().scopeDescriptorForPosition(bufferPosition)
   }
 
   // Extended: Get the range in buffer coordinates of all tokens surrounding the
@@ -3604,7 +3603,7 @@ class TextEditor {
   }
 
   bufferRangeForScopeAtPosition (scopeSelector, position) {
-    return this.tokenizedBuffer.bufferRangeForScopeAtPosition(scopeSelector, position)
+    return this.buffer.getLanguageMode().bufferRangeForScopeAtPosition(scopeSelector, position)
   }
 
   // Extended: Determine if the given row is entirely a comment
@@ -3622,7 +3621,7 @@ class TextEditor {
   }
 
   tokenForBufferPosition (bufferPosition) {
-    return this.tokenizedBuffer.tokenForPosition(bufferPosition)
+    return this.buffer.getLanguageMode().tokenForPosition(bufferPosition)
   }
 
   /*
@@ -3749,7 +3748,7 @@ class TextEditor {
   // level.
   foldCurrentRow () {
     const {row} = this.getCursorBufferPosition()
-    const range = this.tokenizedBuffer.getFoldableRangeContainingPoint(Point(row, Infinity))
+    const range = this.buffer.getLanguageMode().getFoldableRangeContainingPoint(Point(row, Infinity))
     if (range) return this.displayLayer.foldBufferRange(range)
   }
 
@@ -3769,7 +3768,7 @@ class TextEditor {
   foldBufferRow (bufferRow) {
     let position = Point(bufferRow, Infinity)
     while (true) {
-      const foldableRange = this.tokenizedBuffer.getFoldableRangeContainingPoint(position, this.getTabLength())
+      const foldableRange = this.buffer.getLanguageMode().getFoldableRangeContainingPoint(position, this.getTabLength())
       if (foldableRange) {
         const existingFolds = this.displayLayer.foldsIntersectingBufferRange(Range(foldableRange.start, foldableRange.start))
         if (existingFolds.length === 0) {
@@ -3804,7 +3803,7 @@ class TextEditor {
   // Extended: Fold all foldable lines.
   foldAll () {
     this.displayLayer.destroyAllFolds()
-    for (let range of this.tokenizedBuffer.getFoldableRanges(this.getTabLength())) {
+    for (let range of this.buffer.getLanguageMode().getFoldableRanges(this.getTabLength())) {
       this.displayLayer.foldBufferRange(range)
     }
   }
@@ -3821,7 +3820,7 @@ class TextEditor {
   // * `level` A {Number}.
   foldAllAtIndentLevel (level) {
     this.displayLayer.destroyAllFolds()
-    for (let range of this.tokenizedBuffer.getFoldableRangesAtIndentLevel(level, this.getTabLength())) {
+    for (let range of this.buffer.getLanguageMode().getFoldableRangesAtIndentLevel(level, this.getTabLength())) {
       this.displayLayer.foldBufferRange(range)
     }
   }
@@ -3834,7 +3833,7 @@ class TextEditor {
   //
   // Returns a {Boolean}.
   isFoldableAtBufferRow (bufferRow) {
-    return this.tokenizedBuffer.isFoldableAtRow(bufferRow)
+    return this.buffer.getLanguageMode().isFoldableAtRow(bufferRow)
   }
 
   // Extended: Determine whether the given row in screen coordinates is foldable.
@@ -4039,18 +4038,6 @@ class TextEditor {
   Section: Config
   */
 
-  // Experimental: Supply an object that will provide the editor with settings
-  // for specific syntactic scopes. See the `ScopedSettingsDelegate` in
-  // `text-editor-registry.js` for an example implementation.
-  setScopedSettingsDelegate (scopedSettingsDelegate) {
-    this.scopedSettingsDelegate = scopedSettingsDelegate
-    this.tokenizedBuffer.scopedSettingsDelegate = this.scopedSettingsDelegate
-  }
-
-  // Experimental: Retrieve the {Object} that provides the editor with settings
-  // for specific syntactic scopes.
-  getScopedSettingsDelegate () { return this.scopedSettingsDelegate }
-
   // Experimental: Is auto-indentation enabled for this editor?
   //
   // Returns a {Boolean}.
@@ -4099,20 +4086,18 @@ class TextEditor {
   //
   // Returns a {String} containing the non-word characters.
   getNonWordCharacters (scopes) {
-    if (this.scopedSettingsDelegate && this.scopedSettingsDelegate.getNonWordCharacters) {
-      return this.scopedSettingsDelegate.getNonWordCharacters(scopes) || this.nonWordCharacters
-    } else {
-      return this.nonWordCharacters
-    }
+    const languageMode = this.buffer.getLanguageMode()
+    return (languageMode.getNonWordCharacters && languageMode.getNonWordCharacters(scopes)) ||
+      this.nonWordCharacters
   }
 
   /*
   Section: Event Handlers
   */
 
-  handleGrammarChange () {
+  handleLanguageModeChange () {
     this.unfoldAll()
-    return this.emitter.emit('did-change-grammar', this.getGrammar())
+    this.emitter.emit('did-change-grammar', this.buffer.getLanguageMode().getGrammar())
   }
 
   /*
@@ -4382,7 +4367,7 @@ class TextEditor {
   */
 
   suggestedIndentForBufferRow (bufferRow, options) {
-    return this.tokenizedBuffer.suggestedIndentForBufferRow(bufferRow, options)
+    return this.buffer.getLanguageMode().suggestedIndentForBufferRow(bufferRow, options)
   }
 
   // Given a buffer row, indent it.
@@ -4407,7 +4392,7 @@ class TextEditor {
   }
 
   autoDecreaseIndentForBufferRow (bufferRow) {
-    const indentLevel = this.tokenizedBuffer.suggestedIndentForEditedBufferRow(bufferRow)
+    const indentLevel = this.buffer.getLanguageMode().suggestedIndentForEditedBufferRow(bufferRow)
     if (indentLevel != null) this.setIndentationForBufferRow(bufferRow, indentLevel)
   }
 
@@ -4417,7 +4402,7 @@ class TextEditor {
     let {
       commentStartString,
       commentEndString
-    } = this.tokenizedBuffer.commentStringsForPosition(Point(start, 0))
+    } = this.buffer.getLanguageMode().commentStringsForPosition(Point(start, 0))
     if (!commentStartString) return
     commentStartString = commentStartString.trim()
 
@@ -4508,12 +4493,13 @@ class TextEditor {
   rowRangeForParagraphAtBufferRow (bufferRow) {
     if (!NON_WHITESPACE_REGEXP.test(this.lineTextForBufferRow(bufferRow))) return
 
-    const isCommented = this.tokenizedBuffer.isRowCommented(bufferRow)
+    const languageMode = this.buffer.getLanguageMode()
+    const isCommented = languageMode.isRowCommented(bufferRow)
 
     let startRow = bufferRow
     while (startRow > 0) {
       if (!NON_WHITESPACE_REGEXP.test(this.lineTextForBufferRow(startRow - 1))) break
-      if (this.tokenizedBuffer.isRowCommented(startRow - 1) !== isCommented) break
+      if (languageMode.isRowCommented(startRow - 1) !== isCommented) break
       startRow--
     }
 
@@ -4521,7 +4507,7 @@ class TextEditor {
     const rowCount = this.getLineCount()
     while (endRow < rowCount) {
       if (!NON_WHITESPACE_REGEXP.test(this.lineTextForBufferRow(endRow + 1))) break
-      if (this.tokenizedBuffer.isRowCommented(endRow + 1) !== isCommented) break
+      if (languageMode.isRowCommented(endRow + 1) !== isCommented) break
       endRow++
     }
 

src/tokenized-buffer.js

@@ -16,15 +16,6 @@ const prefixedScopes = new Map()
 
 module.exports =
 class TokenizedBuffer {
-  static deserialize (state, atomEnvironment) {
-    const buffer = atomEnvironment.project.bufferForIdSync(state.bufferId)
-    if (!buffer) return null
-
-    state.buffer = buffer
-    state.assert = atomEnvironment.assert
-    return new TokenizedBuffer(state)
-  }
-
   constructor (params) {
     this.emitter = new Emitter()
     this.disposables = new CompositeDisposable()
@@ -37,11 +28,12 @@ class TokenizedBuffer {
     this.buffer = params.buffer
     this.tabLength = params.tabLength
     this.largeFileMode = params.largeFileMode
-    this.assert = params.assert
-    this.scopedSettingsDelegate = params.scopedSettingsDelegate
+    this.config = params.config
 
-    this.setGrammar(params.grammar || NullGrammar)
-    this.disposables.add(this.buffer.registerTextDecorationLayer(this))
+    this.grammar = params.grammar || NullGrammar
+    this.rootScopeDescriptor = new ScopeDescriptor({scopes: [this.grammar.scopeName]})
+    this.disposables.add(this.grammar.onDidUpdate(() => this.retokenizeLines()))
+    this.retokenizeLines()
   }
 
   destroy () {
@@ -59,6 +51,14 @@ class TokenizedBuffer {
     return !this.alive
   }
 
+  getGrammar () {
+    return this.grammar
+  }
+
+  getNonWordCharacters (scope) {
+    return this.config.get('editor.nonWordCharacters', {scope})
+  }
+
   /*
   Section - auto-indent
   */
@@ -164,15 +164,24 @@ class TokenizedBuffer {
   */
 
   commentStringsForPosition (position) {
-    if (this.scopedSettingsDelegate) {
-      const scope = this.scopeDescriptorForPosition(position)
-      return this.scopedSettingsDelegate.getCommentStrings(scope)
-    } else {
-      return {}
+    const scope = this.scopeDescriptorForPosition(position)
+    const commentStartEntries = this.config.getAll('editor.commentStart', {scope})
+    const commentEndEntries = this.config.getAll('editor.commentEnd', {scope})
+    const commentStartEntry = commentStartEntries[0]
+    const commentEndEntry = commentEndEntries.find((entry) => {
+      return entry.scopeSelector === commentStartEntry.scopeSelector
+    })
+    return {
+      commentStartString: commentStartEntry && commentStartEntry.value,
+      commentEndString: commentEndEntry && commentEndEntry.value
     }
   }
 
-  buildIterator () {
+  /*
+  Section - Syntax Highlighting
+  */
+
+  buildHighlightIterator () {
     return new TokenizedBufferIterator(this)
   }
 
@@ -196,47 +205,14 @@ class TokenizedBuffer {
     return []
   }
 
-  onDidInvalidateRange (fn) {
-    return this.emitter.on('did-invalidate-range', fn)
-  }
-
-  serialize () {
-    return {
-      deserializer: 'TokenizedBuffer',
-      bufferPath: this.buffer.getPath(),
-      bufferId: this.buffer.getId(),
-      tabLength: this.tabLength,
-      largeFileMode: this.largeFileMode
-    }
-  }
-
-  observeGrammar (callback) {
-    callback(this.grammar)
-    return this.onDidChangeGrammar(callback)
-  }
-
-  onDidChangeGrammar (callback) {
-    return this.emitter.on('did-change-grammar', callback)
+  onDidChangeHighlighting (fn) {
+    return this.emitter.on('did-change-highlighting', fn)
   }
 
   onDidTokenize (callback) {
     return this.emitter.on('did-tokenize', callback)
   }
 
-  setGrammar (grammar) {
-    if (!grammar || grammar === this.grammar) return
-
-    this.grammar = grammar
-    this.rootScopeDescriptor = new ScopeDescriptor({scopes: [this.grammar.scopeName]})
-
-    if (this.grammarUpdateDisposable) this.grammarUpdateDisposable.dispose()
-    this.grammarUpdateDisposable = this.grammar.onDidUpdate(() => this.retokenizeLines())
-    this.disposables.add(this.grammarUpdateDisposable)
-
-    this.retokenizeLines()
-    this.emitter.emit('did-change-grammar', grammar)
-  }
-
   getGrammarSelectionContent () {
     return this.buffer.getTextInRange([[0, 0], [10, 0]])
   }
@@ -316,7 +292,7 @@ class TokenizedBuffer {
       this.validateRow(endRow)
       if (!filledRegion) this.invalidateRow(endRow + 1)
 
-      this.emitter.emit('did-invalidate-range', Range(Point(startRow, 0), Point(endRow + 1, 0)))
+      this.emitter.emit('did-change-highlighting', Range(Point(startRow, 0), Point(endRow + 1, 0)))
     }
 
     if (this.firstInvalidRow() != null) {
@@ -486,18 +462,6 @@ class TokenizedBuffer {
           while (true) {
             if (scopes.pop() === matchingStartTag) break
             if (scopes.length === 0) {
-              this.assert(false, 'Encountered an unmatched scope end tag.', error => {
-                error.metadata = {
-                  grammarScopeName: this.grammar.scopeName,
-                  unmatchedEndTag: this.grammar.scopeForId(tag)
-                }
-                const path = require('path')
-                error.privateMetadataDescription = `The contents of \`${path.basename(this.buffer.getPath())}\``
-                error.privateMetadata = {
-                  filePath: this.buffer.getPath(),
-                  fileContents: this.buffer.getText()
-                }
-              })
               break
             }
           }
@@ -712,28 +676,20 @@ class TokenizedBuffer {
     return foldEndRow
   }
 
-  increaseIndentRegexForScopeDescriptor (scopeDescriptor) {
-    if (this.scopedSettingsDelegate) {
-      return this.regexForPattern(this.scopedSettingsDelegate.getIncreaseIndentPattern(scopeDescriptor))
-    }
+  increaseIndentRegexForScopeDescriptor (scope) {
+    return this.regexForPattern(this.config.get('editor.increaseIndentPattern', {scope}))
   }
 
-  decreaseIndentRegexForScopeDescriptor (scopeDescriptor) {
-    if (this.scopedSettingsDelegate) {
-      return this.regexForPattern(this.scopedSettingsDelegate.getDecreaseIndentPattern(scopeDescriptor))
-    }
+  decreaseIndentRegexForScopeDescriptor (scope) {
+    return this.regexForPattern(this.config.get('editor.decreaseIndentPattern', {scope}))
   }
 
-  decreaseNextIndentRegexForScopeDescriptor (scopeDescriptor) {
-    if (this.scopedSettingsDelegate) {
-      return this.regexForPattern(this.scopedSettingsDelegate.getDecreaseNextIndentPattern(scopeDescriptor))
-    }
+  decreaseNextIndentRegexForScopeDescriptor (scope) {
+    return this.regexForPattern(this.config.get('editor.decreaseNextIndentPattern', {scope}))
   }
 
-  foldEndRegexForScopeDescriptor (scopes) {
-    if (this.scopedSettingsDelegate) {
-      return this.regexForPattern(this.scopedSettingsDelegate.getFoldEndPattern(scopes))
-    }
+  foldEndRegexForScopeDescriptor (scope) {
+    return this.regexForPattern(this.config.get('editor.foldEndPattern', {scope}))
   }
 
   regexForPattern (pattern) {

src/workspace.js

@@ -494,7 +494,6 @@ module.exports = class Workspace extends Model {
       if (item instanceof TextEditor) {
         const subscriptions = new CompositeDisposable(
           this.textEditorRegistry.add(item),
-          this.textEditorRegistry.maintainGrammar(item),
           this.textEditorRegistry.maintainConfig(item),
           item.observeGrammar(this.handleGrammarUsed.bind(this))
         )
@@ -1250,11 +1249,8 @@ module.exports = class Workspace extends Model {
   // Returns a {TextEditor}.
   buildTextEditor (params) {
     const editor = this.textEditorRegistry.build(params)
-    const subscriptions = new CompositeDisposable(
-      this.textEditorRegistry.maintainGrammar(editor),
-      this.textEditorRegistry.maintainConfig(editor)
-    )
-    editor.onDidDestroy(() => { subscriptions.dispose() })
+    const subscription = this.textEditorRegistry.maintainConfig(editor)
+    editor.onDidDestroy(() => subscription.dispose())
     return editor
   }