Merge pull request #9318 from atom/as-compilation-cache

Introduce NativeCompileCache

package.json

@@ -18,6 +18,7 @@
     "atom-keymap": "^6.1.0",
     "babel-core": "^5.8.21",
     "bootstrap": "^3.3.4",
+    "cached-run-in-this-context": "0.4.0",
     "clear-cut": "^2.0.1",
     "coffee-script": "1.8.0",
     "color": "^0.7.3",

spec/atom-environment-spec.coffee

@@ -215,6 +215,17 @@ describe "AtomEnvironment", ->
       expect(atom.project.getPaths()).toEqual(initialPaths)
 
   describe "::unloadEditorWindow()", ->
+    it "saves the BlobStore so it can be loaded after reload", ->
+      configDirPath = temp.mkdirSync()
+      fakeBlobStore = jasmine.createSpyObj("blob store", ["save"])
+      atomEnvironment = new AtomEnvironment({applicationDelegate: atom.applicationDelegate, enablePersistence: true, configDirPath, blobStore: fakeBlobStore, window, document})
+
+      atomEnvironment.unloadEditorWindow()
+
+      expect(fakeBlobStore.save).toHaveBeenCalled()
+
+      atomEnvironment.destroy()
+
     it "saves the serialized state of the window so it can be deserialized after reload", ->
       atomEnvironment = new AtomEnvironment({applicationDelegate: atom.applicationDelegate, window, document})
       spyOn(atomEnvironment, 'saveStateSync')

spec/file-system-blob-store-spec.coffee

@@ -0,0 +1,69 @@
+temp = require 'temp'
+FileSystemBlobStore = require '../src/file-system-blob-store'
+
+describe "FileSystemBlobStore", ->
+  [storageDirectory, blobStore] = []
+
+  beforeEach ->
+    storageDirectory = temp.path()
+    blobStore = FileSystemBlobStore.load(storageDirectory)
+
+  it "is empty when the file doesn't exist", ->
+    expect(blobStore.get("foo")).toBeUndefined()
+    expect(blobStore.get("bar")).toBeUndefined()
+
+  it "allows to read and write buffers from/to memory without persisting them", ->
+    blobStore.set("foo", new Buffer("foo"))
+    blobStore.set("bar", new Buffer("bar"))
+
+    expect(blobStore.get("foo")).toEqual(new Buffer("foo"))
+    expect(blobStore.get("bar")).toEqual(new Buffer("bar"))
+
+  it "persists buffers when saved and retrieves them on load, giving priority to in-memory ones", ->
+    blobStore.set("foo", new Buffer("foo"))
+    blobStore.set("bar", new Buffer("bar"))
+    blobStore.save()
+
+    blobStore = FileSystemBlobStore.load(storageDirectory)
+
+    expect(blobStore.get("foo")).toEqual(new Buffer("foo"))
+    expect(blobStore.get("bar")).toEqual(new Buffer("bar"))
+
+    blobStore.set("foo", new Buffer("changed"))
+
+    expect(blobStore.get("foo")).toEqual(new Buffer("changed"))
+
+  it "persists both in-memory and previously stored buffers when saved", ->
+    blobStore.set("foo", new Buffer("foo"))
+    blobStore.set("bar", new Buffer("bar"))
+    blobStore.save()
+
+    blobStore = FileSystemBlobStore.load(storageDirectory)
+    blobStore.set("bar", new Buffer("changed"))
+    blobStore.set("qux", new Buffer("qux"))
+    blobStore.save()
+
+    blobStore = FileSystemBlobStore.load(storageDirectory)
+
+    expect(blobStore.get("foo")).toEqual(new Buffer("foo"))
+    expect(blobStore.get("bar")).toEqual(new Buffer("changed"))
+    expect(blobStore.get("qux")).toEqual(new Buffer("qux"))
+
+  it "allows to delete keys from both memory and stored buffers", ->
+    blobStore.set("a", new Buffer("a"))
+    blobStore.set("b", new Buffer("b"))
+    blobStore.save()
+
+    blobStore = FileSystemBlobStore.load(storageDirectory)
+
+    blobStore.set("b", new Buffer("b"))
+    blobStore.set("c", new Buffer("c"))
+    blobStore.delete("b")
+    blobStore.delete("c")
+    blobStore.save()
+
+    blobStore = FileSystemBlobStore.load(storageDirectory)
+
+    expect(blobStore.get("a")).toEqual(new Buffer("a"))
+    expect(blobStore.get("b")).toBeUndefined()
+    expect(blobStore.get("c")).toBeUndefined()

spec/fixtures/native-cache/file-1.js

@@ -0,0 +1 @@
+module.exports = function () { return 1; }

spec/fixtures/native-cache/file-2.js

@@ -0,0 +1 @@
+module.exports = function () { return 2; }

spec/fixtures/native-cache/file-3.js

@@ -0,0 +1 @@
+module.exports = function () { return 3; }

spec/native-compile-cache-spec.coffee

@@ -0,0 +1,47 @@
+describe "NativeCompileCache", ->
+  nativeCompileCache = require '../src/native-compile-cache'
+  [fakeCacheStore, cachedFiles] = []
+
+  beforeEach ->
+    cachedFiles = []
+    fakeCacheStore = jasmine.createSpyObj("cache store", ["set", "get", "has", "delete"])
+    nativeCompileCache.setCacheStore(fakeCacheStore)
+    nativeCompileCache.install()
+
+  it "writes and reads from the cache storage when requiring files", ->
+    fakeCacheStore.has.andReturn(false)
+    fakeCacheStore.set.andCallFake (filename, cacheBuffer) ->
+      cachedFiles.push({filename, cacheBuffer})
+
+    fn1 = require('./fixtures/native-cache/file-1')
+    fn2 = require('./fixtures/native-cache/file-2')
+
+    expect(cachedFiles.length).toBe(2)
+
+    expect(cachedFiles[0].filename).toBe(require.resolve('./fixtures/native-cache/file-1'))
+    expect(cachedFiles[0].cacheBuffer).toBeInstanceOf(Uint8Array)
+    expect(cachedFiles[0].cacheBuffer.length).toBeGreaterThan(0)
+    expect(fn1()).toBe(1)
+
+    expect(cachedFiles[1].filename).toBe(require.resolve('./fixtures/native-cache/file-2'))
+    expect(cachedFiles[1].cacheBuffer).toBeInstanceOf(Uint8Array)
+    expect(cachedFiles[1].cacheBuffer.length).toBeGreaterThan(0)
+    expect(fn2()).toBe(2)
+
+    fakeCacheStore.has.andReturn(true)
+    fakeCacheStore.get.andReturn(cachedFiles[0].cacheBuffer)
+    fakeCacheStore.set.reset()
+
+    fn1 = require('./fixtures/native-cache/file-1')
+
+    expect(fakeCacheStore.set).not.toHaveBeenCalled()
+    expect(fn1()).toBe(1)
+
+  it "deletes previously cached code when the cache is not valid", ->
+    fakeCacheStore.has.andReturn(true)
+    fakeCacheStore.get.andCallFake -> new Buffer("an invalid cache")
+
+    fn3 = require('./fixtures/native-cache/file-3')
+
+    expect(fakeCacheStore.delete).toHaveBeenCalledWith(require.resolve('./fixtures/native-cache/file-3'))
+    expect(fn3()).toBe(3)

src/atom-environment.coffee

@@ -116,7 +116,7 @@ class AtomEnvironment extends Model
 
   # Call .loadOrCreate instead
   constructor: (params={}) ->
-    {@applicationDelegate, @window, @document, configDirPath, @enablePersistence} = params
+    {@blobStore, @applicationDelegate, @window, @document, configDirPath, @enablePersistence} = params
 
     @state = {version: @constructor.version}
 
@@ -306,6 +306,7 @@ class AtomEnvironment extends Model
     @project = null
     @commands.clear()
     @stylesElement.remove()
+    @config.unobserveUserConfig()
 
     @uninstallWindowEventHandler()
 
@@ -636,6 +637,7 @@ class AtomEnvironment extends Model
     @state.packageStates = @packages.packageStates
     @state.fullScreen = @isFullScreen()
     @saveStateSync()
+    @saveBlobStoreSync()
 
   openInitialEmptyEditorIfNecessary: ->
     return unless @config.get('core.openEmptyEditorOnStart')
@@ -759,6 +761,11 @@ class AtomEnvironment extends Model
   showSaveDialogSync: (options={}) ->
     @applicationDelegate.showSaveDialog(options)
 
+  saveBlobStoreSync: ->
+    return unless @enablePersistence
+
+    @blobStore.save()
+
   saveStateSync: ->
     return unless @enablePersistence
 

src/file-system-blob-store.js

@@ -0,0 +1,111 @@
+'use strict'
+
+const fs = require('fs-plus')
+const path = require('path')
+
+module.exports =
+class FileSystemBlobStore {
+  static load (directory) {
+    let instance = new FileSystemBlobStore(directory)
+    instance.load()
+    return instance
+  }
+
+  constructor (directory) {
+    this.inMemoryBlobs = new Map()
+    this.blobFilename = path.join(directory, 'BLOB')
+    this.blobMapFilename = path.join(directory, 'MAP')
+    this.lockFilename = path.join(directory, 'LOCK')
+    this.storedBlob = new Buffer(0)
+    this.storedBlobMap = {}
+  }
+
+  load () {
+    if (!fs.existsSync(this.blobMapFilename)) {
+      return
+    }
+    if (!fs.existsSync(this.blobFilename)) {
+      return
+    }
+    this.storedBlob = fs.readFileSync(this.blobFilename)
+    this.storedBlobMap = JSON.parse(fs.readFileSync(this.blobMapFilename))
+  }
+
+  save () {
+    let dump = this.getDump()
+    let blobToStore = Buffer.concat(dump[0])
+    let mapToStore = JSON.stringify(dump[1])
+
+    let acquiredLock = false
+    try {
+      fs.writeFileSync(this.lockFilename, 'LOCK', {flag: 'wx'})
+      acquiredLock = true
+
+      fs.writeFileSync(this.blobFilename, blobToStore)
+      fs.writeFileSync(this.blobMapFilename, mapToStore)
+    } catch (error) {
+      // Swallow the exception silently only if we fail to acquire the lock.
+      if (error.code !== 'EEXIST') {
+        throw error
+      }
+    } finally {
+      if (acquiredLock) {
+        fs.unlinkSync(this.lockFilename)
+      }
+    }
+  }
+
+  has (key) {
+    return this.inMemoryBlobs.hasOwnProperty(key) || this.storedBlobMap.hasOwnProperty(key)
+  }
+
+  get (key) {
+    return this.getFromMemory(key) || this.getFromStorage(key)
+  }
+
+  set (key, buffer) {
+    return this.inMemoryBlobs.set(key, buffer)
+  }
+
+  delete (key) {
+    this.inMemoryBlobs.delete(key)
+    delete this.storedBlobMap[key]
+  }
+
+  getFromMemory (key) {
+    return this.inMemoryBlobs.get(key)
+  }
+
+  getFromStorage (key) {
+    if (!this.storedBlobMap[key]) {
+      return
+    }
+
+    return this.storedBlob.slice.apply(this.storedBlob, this.storedBlobMap[key])
+  }
+
+  getDump () {
+    let buffers = []
+    let blobMap = {}
+    let currentBufferStart = 0
+
+    function dump (key, getBufferByKey) {
+      let buffer = getBufferByKey(key)
+      buffers.push(buffer)
+      blobMap[key] = [currentBufferStart, currentBufferStart + buffer.length]
+      currentBufferStart += buffer.length
+    }
+
+    for (let key of this.inMemoryBlobs.keys()) {
+      dump(key, this.getFromMemory.bind(this))
+    }
+
+    for (let key of Object.keys(this.storedBlobMap)) {
+      if (!blobMap[key]) {
+        dump(key, this.getFromStorage.bind(this))
+      }
+    }
+
+    return [buffers, blobMap]
+  }
+}

src/initialize-application-window.coffee

@@ -1,34 +1,34 @@
 # Like sands through the hourglass, so are the days of our lives.
+module.exports = ({blobStore}) ->
+  path = require 'path'
+  require './window'
+  {getWindowLoadSettings} = require './window-load-settings-helpers'
 
-path = require 'path'
-require './window'
-{getWindowLoadSettings} = require './window-load-settings-helpers'
+  {resourcePath, isSpec, devMode} = getWindowLoadSettings()
 
-{resourcePath, isSpec, devMode} = getWindowLoadSettings()
+  # Add application-specific exports to module search path.
+  exportsPath = path.join(resourcePath, 'exports')
+  require('module').globalPaths.push(exportsPath)
+  process.env.NODE_PATH = exportsPath
 
-# Add application-specific exports to module search path.
-exportsPath = path.join(resourcePath, 'exports')
-require('module').globalPaths.push(exportsPath)
-process.env.NODE_PATH = exportsPath
+  # Make React faster
+  process.env.NODE_ENV ?= 'production' unless devMode
 
-# Make React faster
-process.env.NODE_ENV ?= 'production' unless devMode
+  AtomEnvironment = require './atom-environment'
+  ApplicationDelegate = require './application-delegate'
+  window.atom = new AtomEnvironment({
+    window, document, blobStore,
+    applicationDelegate: new ApplicationDelegate,
+    configDirPath: process.env.ATOM_HOME
+    enablePersistence: true
+  })
 
-AtomEnvironment = require './atom-environment'
-ApplicationDelegate = require './application-delegate'
-window.atom = new AtomEnvironment({
-  window, document,
-  applicationDelegate: new ApplicationDelegate,
-  configDirPath: process.env.ATOM_HOME
-  enablePersistence: true
-})
+  atom.displayWindow()
+  atom.loadStateSync()
+  atom.startEditorWindow()
 
-atom.displayWindow()
-atom.loadStateSync()
-atom.startEditorWindow()
-
-# Workaround for focus getting cleared upon window creation
-windowFocused = ->
-  window.removeEventListener('focus', windowFocused)
-  setTimeout (-> document.querySelector('atom-workspace').focus()), 0
-window.addEventListener('focus', windowFocused)
+  # Workaround for focus getting cleared upon window creation
+  windowFocused = ->
+    window.removeEventListener('focus', windowFocused)
+    setTimeout (-> document.querySelector('atom-workspace').focus()), 0
+  window.addEventListener('focus', windowFocused)

src/initialize-test-window.coffee

@@ -1,69 +1,78 @@
-# Start the crash reporter before anything else.
-require('crash-reporter').start(productName: 'Atom', companyName: 'GitHub')
-remote = require 'remote'
+cloneObject = (object) ->
+  clone = {}
+  clone[key] = value for key, value of object
+  clone
 
-exitWithStatusCode = (status) ->
-  remote.require('app').emit('will-quit')
-  remote.process.exit(status)
+module.exports = ({blobStore}) ->
+  # Start the crash reporter before anything else.
+  require('crash-reporter').start(productName: 'Atom', companyName: 'GitHub')
+  remote = require 'remote'
 
-try
-  path = require 'path'
-  ipc = require 'ipc'
-  {getWindowLoadSettings} = require './window-load-settings-helpers'
-  AtomEnvironment = require '../src/atom-environment'
-  ApplicationDelegate = require '../src/application-delegate'
+  exitWithStatusCode = (status) ->
+    remote.require('app').emit('will-quit')
+    remote.process.exit(status)
 
-  {testRunnerPath, legacyTestRunnerPath, headless, logFile, testPaths} = getWindowLoadSettings()
+  try
+    path = require 'path'
+    ipc = require 'ipc'
+    {getWindowLoadSettings} = require './window-load-settings-helpers'
+    AtomEnvironment = require '../src/atom-environment'
+    ApplicationDelegate = require '../src/application-delegate'
 
-  if headless
-    # Override logging in headless mode so it goes to the console, regardless
-    # of the --enable-logging flag to Electron.
-    console.log = (args...) ->
-      ipc.send 'write-to-stdout', args.join(' ') + '\n'
-    console.warn = (args...) ->
-      ipc.send 'write-to-stderr', args.join(' ') + '\n'
-    console.error = (args...) ->
-      ipc.send 'write-to-stderr', args.join(' ') + '\n'
-  else
-    # Show window synchronously so a focusout doesn't fire on input elements
-    # that are focused in the very first spec run.
-    remote.getCurrentWindow().show()
+    {testRunnerPath, legacyTestRunnerPath, headless, logFile, testPaths} = getWindowLoadSettings()
 
-  handleKeydown = (event) ->
-    # Reload: cmd-r / ctrl-r
-    if (event.metaKey or event.ctrlKey) and event.keyCode is 82
-      ipc.send('call-window-method', 'restart')
+    if headless
+      # Override logging in headless mode so it goes to the console, regardless
+      # of the --enable-logging flag to Electron.
+      console.log = (args...) ->
+        ipc.send 'write-to-stdout', args.join(' ') + '\n'
+      console.warn = (args...) ->
+        ipc.send 'write-to-stderr', args.join(' ') + '\n'
+      console.error = (args...) ->
+        ipc.send 'write-to-stderr', args.join(' ') + '\n'
+    else
+      # Show window synchronously so a focusout doesn't fire on input elements
+      # that are focused in the very first spec run.
+      remote.getCurrentWindow().show()
 
-    # Toggle Dev Tools: cmd-alt-i / ctrl-alt-i
-    if (event.metaKey or event.ctrlKey) and event.altKey and event.keyCode is 73
-      ipc.send('call-window-method', 'toggleDevTools')
+    handleKeydown = (event) ->
+      # Reload: cmd-r / ctrl-r
+      if (event.metaKey or event.ctrlKey) and event.keyCode is 82
+        ipc.send('call-window-method', 'restart')
 
-    # Reload: cmd-w / ctrl-w
-    if (event.metaKey or event.ctrlKey) and event.keyCode is 87
-      ipc.send('call-window-method', 'close')
+      # Toggle Dev Tools: cmd-alt-i / ctrl-alt-i
+      if (event.metaKey or event.ctrlKey) and event.altKey and event.keyCode is 73
+        ipc.send('call-window-method', 'toggleDevTools')
 
-  window.addEventListener('keydown', handleKeydown, true)
+      # Reload: cmd-w / ctrl-w
+      if (event.metaKey or event.ctrlKey) and event.keyCode is 87
+        ipc.send('call-window-method', 'close')
 
-  # Add 'exports' to module search path.
-  exportsPath = path.join(getWindowLoadSettings().resourcePath, 'exports')
-  require('module').globalPaths.push(exportsPath)
-  process.env.NODE_PATH = exportsPath # Set NODE_PATH env variable since tasks may need it.
+    window.addEventListener('keydown', handleKeydown, true)
 
-  document.title = "Spec Suite"
+    # Add 'exports' to module search path.
+    exportsPath = path.join(getWindowLoadSettings().resourcePath, 'exports')
+    require('module').globalPaths.push(exportsPath)
+    process.env.NODE_PATH = exportsPath # Set NODE_PATH env variable since tasks may need it.
 
-  testRunner = require(testRunnerPath)
-  legacyTestRunner = require(legacyTestRunnerPath)
-  buildAtomEnvironment = (params) -> new AtomEnvironment(params)
-  buildDefaultApplicationDelegate = (params) -> new ApplicationDelegate()
+    document.title = "Spec Suite"
 
-  promise = testRunner({
-    logFile, headless, testPaths, buildAtomEnvironment, buildDefaultApplicationDelegate, legacyTestRunner
-  })
+    testRunner = require(testRunnerPath)
+    legacyTestRunner = require(legacyTestRunnerPath)
+    buildDefaultApplicationDelegate = -> new ApplicationDelegate()
+    buildAtomEnvironment = (params) ->
+      params = cloneObject(params)
+      params.blobStore = blobStore unless params.hasOwnProperty("blobStore")
+      new AtomEnvironment(params)
 
-  promise.then(exitWithStatusCode) if getWindowLoadSettings().headless
-catch error
-  if getWindowLoadSettings().headless
-    console.error(error.stack ? error)
-    exitWithStatusCode(1)
-  else
-    throw error
+    promise = testRunner({
+      logFile, headless, testPaths, buildAtomEnvironment, buildDefaultApplicationDelegate, legacyTestRunner
+    })
+
+    promise.then(exitWithStatusCode) if getWindowLoadSettings().headless
+  catch error
+    if getWindowLoadSettings().headless
+      console.error(error.stack ? error)
+      exitWithStatusCode(1)
+    else
+      throw error

src/native-compile-cache.js

@@ -0,0 +1,101 @@
+'use strict'
+
+const Module = require('module')
+const path = require('path')
+const cachedVm = require('cached-run-in-this-context')
+
+class NativeCompileCache {
+  constructor () {
+    this.cacheStore = null
+    this.previousModuleCompile = null
+  }
+
+  setCacheStore (store) {
+    this.cacheStore = store
+  }
+
+  install () {
+    this.savePreviousModuleCompile()
+    this.overrideModuleCompile()
+  }
+
+  uninstall () {
+    this.restorePreviousModuleCompile()
+  }
+
+  savePreviousModuleCompile () {
+    this.previousModuleCompile = Module.prototype._compile
+  }
+
+  overrideModuleCompile () {
+    let cacheStore = this.cacheStore
+    let resolvedArgv = null
+    // Here we override Node's module.js
+    // (https://github.com/atom/node/blob/atom/lib/module.js#L378), changing
+    // only the bits that affect compilation in order to use the cached one.
+    Module.prototype._compile = function (content, filename) {
+      let self = this
+      // remove shebang
+      content = content.replace(/^\#\!.*/, '')
+      function require (path) {
+        return self.require(path)
+      }
+      require.resolve = function (request) {
+        return Module._resolveFilename(request, self)
+      }
+      require.main = process.mainModule
+
+      // Enable support to add extra extension types
+      require.extensions = Module._extensions
+      require.cache = Module._cache
+
+      let dirname = path.dirname(filename)
+
+      // create wrapper function
+      let wrapper = Module.wrap(content)
+
+      let compiledWrapper = null
+      if (cacheStore.has(filename)) {
+        let buffer = cacheStore.get(filename)
+        let compilationResult = cachedVm.runInThisContextCached(wrapper, filename, buffer)
+        compiledWrapper = compilationResult.result
+        if (compilationResult.wasRejected) {
+          cacheStore.delete(filename)
+        }
+      } else {
+        let compilationResult = cachedVm.runInThisContext(wrapper, filename)
+        if (compilationResult.cacheBuffer) {
+          cacheStore.set(filename, compilationResult.cacheBuffer)
+        }
+        compiledWrapper = compilationResult.result
+      }
+      if (global.v8debug) {
+        if (!resolvedArgv) {
+          // we enter the repl if we're not given a filename argument.
+          if (process.argv[1]) {
+            resolvedArgv = Module._resolveFilename(process.argv[1], null)
+          } else {
+            resolvedArgv = 'repl'
+          }
+        }
+
+        // Set breakpoint on module start
+        if (filename === resolvedArgv) {
+          // Installing this dummy debug event listener tells V8 to start
+          // the debugger.  Without it, the setBreakPoint() fails with an
+          // 'illegal access' error.
+          global.v8debug.Debug.setListener(function () {})
+          global.v8debug.Debug.setBreakPoint(compiledWrapper, 0, 0)
+        }
+      }
+      let args = [self.exports, require, self, filename, dirname, process, global]
+      return compiledWrapper.apply(self.exports, args)
+    }
+  }
+
+  restorePreviousModuleCompile () {
+    Module.prototype._compile = this.previousModuleCompile
+  }
+}
+
+module.exports = new NativeCompileCache()

static/index.js

@@ -1,9 +1,12 @@
 (function () {
-  var fs = require('fs')
+  var fs = require('fs-plus')
   var path = require('path')
+  var FileSystemBlobStore = require('../src/file-system-blob-store')
+  var NativeCompileCache = require('../src/native-compile-cache')
 
   var loadSettings = null
   var loadSettingsError = null
+  var blobStore = null
 
   window.onload = function () {
     try {
@@ -16,6 +19,12 @@
       // Ensure ATOM_HOME is always set before anything else is required
       setupAtomHome()
 
+      blobStore = FileSystemBlobStore.load(
+        path.join(process.env.ATOM_HOME, 'blob-store/')
+      )
+      NativeCompileCache.setCacheStore(blobStore)
+      NativeCompileCache.install()
+
       // Normalize to make sure drive letter case is consistent on Windows
       process.resourcesPath = path.normalize(process.resourcesPath)
 
@@ -76,7 +85,8 @@
     setupVmCompatibility()
     setupCsonCache(CompileCache.getCacheDirectory())
 
-    require(loadSettings.windowInitializationScript)
+    var initialize = require(loadSettings.windowInitializationScript)
+    initialize({blobStore: blobStore})
     require('ipc').sendChannel('window-command', 'window:loaded')
   }