Re-enable native module cache

src/atom-environment.coffee

@@ -131,7 +131,7 @@ class AtomEnvironment extends Model
 
   # Call .loadOrCreate instead
   constructor: (params={}) ->
-    {@applicationDelegate, @window, @document, @clipboard, @configDirPath, @enablePersistence, onlyLoadBaseStyleSheets} = params
+    {@applicationDelegate, @window, @document, @blobStore, @clipboard, @configDirPath, @enablePersistence, onlyLoadBaseStyleSheets} = params
 
     @unloaded = false
     @loadTime = null
@@ -733,8 +733,13 @@ class AtomEnvironment extends Model
 
     @storeWindowBackground()
     @packages.deactivatePackages()
+    @saveBlobStoreSync()
     @unloaded = true
 
+  saveBlobStoreSync: ->
+    if @enablePersistence
+      @blobStore.save()
+
   openInitialEmptyEditorIfNecessary: ->
     return unless @config.get('core.openEmptyEditorOnStart')
     if @getLoadSettings().initialPaths?.length is 0 and @workspace.getPaneItems().length is 0

src/file-system-blob-store.js

@@ -0,0 +1,138 @@
+'use strict'
+
+const fs = require('fs-plus')
+const path = require('path')
+
+module.exports =
+class FileSystemBlobStore {
+  static load (directory) {
+    let instance = new FileSystemBlobStore(directory)
+    instance.load()
+    return instance
+  }
+
+  constructor (directory) {
+    this.blobFilename = path.join(directory, 'BLOB')
+    this.blobMapFilename = path.join(directory, 'MAP')
+    this.invalidationKeysFilename = path.join(directory, 'INVKEYS')
+    this.lockFilename = path.join(directory, 'LOCK')
+    this.reset()
+  }
+
+  reset () {
+    this.inMemoryBlobs = new Map()
+    this.invalidationKeys = {}
+    this.storedBlob = new Buffer(0)
+    this.storedBlobMap = {}
+    this.usedKeys = new Set()
+  }
+
+  load () {
+    if (!fs.existsSync(this.blobMapFilename)) {
+      return
+    }
+    if (!fs.existsSync(this.blobFilename)) {
+      return
+    }
+    if (!fs.existsSync(this.invalidationKeysFilename)) {
+      return
+    }
+
+    try {
+      this.storedBlob = fs.readFileSync(this.blobFilename)
+      this.storedBlobMap = JSON.parse(fs.readFileSync(this.blobMapFilename))
+      this.invalidationKeys = JSON.parse(fs.readFileSync(this.invalidationKeysFilename))
+    } catch (e) {
+      this.reset()
+    }
+  }
+
+  save () {
+    let dump = this.getDump()
+    let blobToStore = Buffer.concat(dump[0])
+    let mapToStore = JSON.stringify(dump[1])
+    let invalidationKeysToStore = JSON.stringify(this.invalidationKeys)
+
+    let acquiredLock = false
+    try {
+      fs.writeFileSync(this.lockFilename, 'LOCK', {flag: 'wx'})
+      acquiredLock = true
+
+      fs.writeFileSync(this.blobFilename, blobToStore)
+      fs.writeFileSync(this.blobMapFilename, mapToStore)
+      fs.writeFileSync(this.invalidationKeysFilename, invalidationKeysToStore)
+    } catch (error) {
+      // Swallow the exception silently only if we fail to acquire the lock.
+      if (error.code !== 'EEXIST') {
+        throw error
+      }
+    } finally {
+      if (acquiredLock) {
+        fs.unlinkSync(this.lockFilename)
+      }
+    }
+  }
+
+  has (key, invalidationKey) {
+    let containsKey = this.inMemoryBlobs.has(key) || this.storedBlobMap.hasOwnProperty(key)
+    let isValid = this.invalidationKeys[key] === invalidationKey
+    return containsKey && isValid
+  }
+
+  get (key, invalidationKey) {
+    if (this.has(key, invalidationKey)) {
+      this.usedKeys.add(key)
+      return this.getFromMemory(key) || this.getFromStorage(key)
+    }
+  }
+
+  set (key, invalidationKey, buffer) {
+    this.usedKeys.add(key)
+    this.invalidationKeys[key] = invalidationKey
+    return this.inMemoryBlobs.set(key, buffer)
+  }
+
+  delete (key) {
+    this.inMemoryBlobs.delete(key)
+    delete this.storedBlobMap[key]
+  }
+
+  getFromMemory (key) {
+    return this.inMemoryBlobs.get(key)
+  }
+
+  getFromStorage (key) {
+    if (!this.storedBlobMap[key]) {
+      return
+    }
+
+    return this.storedBlob.slice.apply(this.storedBlob, this.storedBlobMap[key])
+  }
+
+  getDump () {
+    let buffers = []
+    let blobMap = {}
+    let currentBufferStart = 0
+
+    function dump (key, getBufferByKey) {
+      let buffer = getBufferByKey(key)
+      buffers.push(buffer)
+      blobMap[key] = [currentBufferStart, currentBufferStart + buffer.length]
+      currentBufferStart += buffer.length
+    }
+
+    for (let key of this.inMemoryBlobs.keys()) {
+      if (this.usedKeys.has(key)) {
+        dump(key, this.getFromMemory.bind(this))
+      }
+    }
+
+    for (let key of Object.keys(this.storedBlobMap)) {
+      if (!blobMap[key] && this.usedKeys.has(key)) {
+        dump(key, this.getFromStorage.bind(this))
+      }
+    }
+
+    return [buffers, blobMap]
+  }
+}

src/initialize-application-window.coffee

@@ -3,6 +3,7 @@ ApplicationDelegate = require './application-delegate'
 Clipboard = require './clipboard'
 TextEditor = require './text-editor'
 TextEditorComponent = require './text-editor-component'
+FileSystemBlobStore = require './file-system-blob-store'
 CompileCache = require './compile-cache'
 ModuleCache = require './module-cache'
 
@@ -54,7 +55,7 @@ require('whitespace')
 require('wrap-guide')
 
 # Like sands through the hourglass, so are the days of our lives.
-module.exports = ->
+module.exports = ({blobStore}) ->
   {updateProcessEnv} = require('./update-process-env')
   path = require 'path'
   require './window'
@@ -75,7 +76,7 @@ module.exports = ->
   TextEditor.setClipboard(clipboard)
 
   window.atom = new AtomEnvironment({
-    window, document, clipboard,
+    window, document, clipboard, blobStore,
     applicationDelegate: new ApplicationDelegate,
     configDirPath: process.env.ATOM_HOME,
     enablePersistence: true,

src/initialize-test-window.coffee

@@ -5,7 +5,7 @@ cloneObject = (object) ->
   clone[key] = value for key, value of object
   clone
 
-module.exports = ->
+module.exports = ({blobStore}) ->
   startCrashReporter = require('./crash-reporter-start')
   {remote} = require 'electron'
 
@@ -77,6 +77,7 @@ module.exports = ->
     buildAtomEnvironment = (params) ->
       params = cloneObject(params)
       params.clipboard = clipboard unless params.hasOwnProperty("clipboard")
+      params.blobStore = blobStore unless params.hasOwnProperty("blobStore")
       params.onlyLoadBaseStyleSheets = true unless params.hasOwnProperty("onlyLoadBaseStyleSheets")
       new AtomEnvironment(params)
 

src/native-compile-cache.js

@@ -0,0 +1,116 @@
+const Module = require('module')
+const path = require('path')
+const cachedVm = require('cached-run-in-this-context')
+const crypto = require('crypto')
+
+function computeHash (contents) {
+  return crypto.createHash('sha1').update(contents, 'utf8').digest('hex')
+}
+
+class NativeCompileCache {
+  constructor () {
+    this.cacheStore = null
+    this.previousModuleCompile = null
+  }
+
+  setCacheStore (store) {
+    this.cacheStore = store
+  }
+
+  setV8Version (v8Version) {
+    this.v8Version = v8Version.toString()
+  }
+
+  install () {
+    this.savePreviousModuleCompile()
+    this.overrideModuleCompile()
+  }
+
+  uninstall () {
+    this.restorePreviousModuleCompile()
+  }
+
+  savePreviousModuleCompile () {
+    this.previousModuleCompile = Module.prototype._compile
+  }
+
+  overrideModuleCompile () {
+    let self = this
+    let resolvedArgv = null
+    // Here we override Node's module.js
+    // (https://github.com/atom/node/blob/atom/lib/module.js#L378), changing
+    // only the bits that affect compilation in order to use the cached one.
+    Module.prototype._compile = function (content, filename) {
+      let moduleSelf = this
+      // remove shebang
+      content = content.replace(/^#!.*/, '')
+      function require (path) {
+        return moduleSelf.require(path)
+      }
+      require.resolve = function (request) {
+        return Module._resolveFilename(request, moduleSelf)
+      }
+      require.main = process.mainModule
+
+      // Enable support to add extra extension types
+      require.extensions = Module._extensions
+      require.cache = Module._cache
+
+      let dirname = path.dirname(filename)
+
+      // create wrapper function
+      let wrapper = Module.wrap(content)
+
+      let cacheKey = filename
+      let invalidationKey = computeHash(wrapper + self.v8Version)
+      let compiledWrapper = null
+      if (self.cacheStore.has(cacheKey, invalidationKey)) {
+        let buffer = self.cacheStore.get(cacheKey, invalidationKey)
+        let compilationResult = cachedVm.runInThisContextCached(wrapper, filename, buffer)
+        compiledWrapper = compilationResult.result
+        if (compilationResult.wasRejected) {
+          self.cacheStore.delete(cacheKey)
+        }
+      } else {
+        let compilationResult
+        try {
+          compilationResult = cachedVm.runInThisContext(wrapper, filename)
+        } catch (err) {
+          console.error(`Error running script ${filename}`)
+          throw err
+        }
+        if (compilationResult.cacheBuffer) {
+          self.cacheStore.set(cacheKey, invalidationKey, compilationResult.cacheBuffer)
+        }
+        compiledWrapper = compilationResult.result
+      }
+      if (global.v8debug) {
+        if (!resolvedArgv) {
+          // we enter the repl if we're not given a filename argument.
+          if (process.argv[1]) {
+            resolvedArgv = Module._resolveFilename(process.argv[1], null)
+          } else {
+            resolvedArgv = 'repl'
+          }
+        }
+
+        // Set breakpoint on module start
+        if (filename === resolvedArgv) {
+          // Installing this dummy debug event listener tells V8 to start
+          // the debugger.  Without it, the setBreakPoint() fails with an
+          // 'illegal access' error.
+          global.v8debug.Debug.setListener(function () {})
+          global.v8debug.Debug.setBreakPoint(compiledWrapper, 0, 0)
+        }
+      }
+      let args = [moduleSelf.exports, require, moduleSelf, filename, dirname, process, global]
+      return compiledWrapper.apply(moduleSelf.exports, args)
+    }
+  }
+
+  restorePreviousModuleCompile () {
+    Module.prototype._compile = this.previousModuleCompile
+  }
+}
+
+module.exports = new NativeCompileCache()