diff --git a/spec/file-system-blob-store-spec.coffee b/spec/file-system-blob-store-spec.coffee new file mode 100644 index 000000000..70e4d2b8d --- /dev/null +++ b/spec/file-system-blob-store-spec.coffee @@ -0,0 +1,109 @@ +temp = require('temp').track() +path = require 'path' +fs = require 'fs-plus' +FileSystemBlobStore = require '../src/file-system-blob-store' + +describe "FileSystemBlobStore", -> + [storageDirectory, blobStore] = [] + + beforeEach -> + storageDirectory = temp.path('atom-spec-filesystemblobstore') + blobStore = FileSystemBlobStore.load(storageDirectory) + + afterEach -> + fs.removeSync(storageDirectory) + + it "is empty when the file doesn't exist", -> + expect(blobStore.get("foo", "invalidation-key-1")).toBeUndefined() + expect(blobStore.get("bar", "invalidation-key-2")).toBeUndefined() + + it "allows to read and write buffers from/to memory without persisting them", -> + blobStore.set("foo", "invalidation-key-1", new Buffer("foo")) + blobStore.set("bar", "invalidation-key-2", new Buffer("bar")) + + expect(blobStore.get("foo", "invalidation-key-1")).toEqual(new Buffer("foo")) + expect(blobStore.get("bar", "invalidation-key-2")).toEqual(new Buffer("bar")) + + expect(blobStore.get("foo", "unexisting-key")).toBeUndefined() + expect(blobStore.get("bar", "unexisting-key")).toBeUndefined() + + it "persists buffers when saved and retrieves them on load, giving priority to in-memory ones", -> + blobStore.set("foo", "invalidation-key-1", new Buffer("foo")) + blobStore.set("bar", "invalidation-key-2", new Buffer("bar")) + blobStore.save() + + blobStore = FileSystemBlobStore.load(storageDirectory) + + expect(blobStore.get("foo", "invalidation-key-1")).toEqual(new Buffer("foo")) + expect(blobStore.get("bar", "invalidation-key-2")).toEqual(new Buffer("bar")) + expect(blobStore.get("foo", "unexisting-key")).toBeUndefined() + expect(blobStore.get("bar", "unexisting-key")).toBeUndefined() + + blobStore.set("foo", "new-key", new Buffer("changed")) + + expect(blobStore.get("foo", "new-key")).toEqual(new Buffer("changed")) + expect(blobStore.get("foo", "invalidation-key-1")).toBeUndefined() + + it "persists in-memory and previously stored buffers, and deletes unused keys when saved", -> + blobStore.set("foo", "invalidation-key-1", new Buffer("foo")) + blobStore.set("bar", "invalidation-key-2", new Buffer("bar")) + blobStore.save() + + blobStore = FileSystemBlobStore.load(storageDirectory) + blobStore.set("bar", "invalidation-key-3", new Buffer("changed")) + blobStore.set("qux", "invalidation-key-4", new Buffer("qux")) + blobStore.save() + + blobStore = FileSystemBlobStore.load(storageDirectory) + + expect(blobStore.get("foo", "invalidation-key-1")).toBeUndefined() + expect(blobStore.get("bar", "invalidation-key-3")).toEqual(new Buffer("changed")) + expect(blobStore.get("qux", "invalidation-key-4")).toEqual(new Buffer("qux")) + expect(blobStore.get("foo", "unexisting-key")).toBeUndefined() + expect(blobStore.get("bar", "invalidation-key-2")).toBeUndefined() + expect(blobStore.get("qux", "unexisting-key")).toBeUndefined() + + it "allows to delete keys from both memory and stored buffers", -> + blobStore.set("a", "invalidation-key-1", new Buffer("a")) + blobStore.set("b", "invalidation-key-2", new Buffer("b")) + blobStore.save() + + blobStore = FileSystemBlobStore.load(storageDirectory) + + blobStore.get("a", "invalidation-key-1") # prevent the key from being deleted on save + blobStore.set("b", "invalidation-key-3", new Buffer("b")) + blobStore.set("c", "invalidation-key-4", new Buffer("c")) + blobStore.delete("b") + blobStore.delete("c") + blobStore.save() + + blobStore = FileSystemBlobStore.load(storageDirectory) + + expect(blobStore.get("a", "invalidation-key-1")).toEqual(new Buffer("a")) + expect(blobStore.get("b", "invalidation-key-2")).toBeUndefined() + expect(blobStore.get("b", "invalidation-key-3")).toBeUndefined() + expect(blobStore.get("c", "invalidation-key-4")).toBeUndefined() + + it "ignores errors when loading an invalid blob store", -> + blobStore.set("a", "invalidation-key-1", new Buffer("a")) + blobStore.set("b", "invalidation-key-2", new Buffer("b")) + blobStore.save() + + # Simulate corruption + fs.writeFileSync(path.join(storageDirectory, "MAP"), new Buffer([0])) + fs.writeFileSync(path.join(storageDirectory, "INVKEYS"), new Buffer([0])) + fs.writeFileSync(path.join(storageDirectory, "BLOB"), new Buffer([0])) + + blobStore = FileSystemBlobStore.load(storageDirectory) + + expect(blobStore.get("a", "invalidation-key-1")).toBeUndefined() + expect(blobStore.get("b", "invalidation-key-2")).toBeUndefined() + + blobStore.set("a", "invalidation-key-1", new Buffer("x")) + blobStore.set("b", "invalidation-key-2", new Buffer("y")) + blobStore.save() + + blobStore = FileSystemBlobStore.load(storageDirectory) + + expect(blobStore.get("a", "invalidation-key-1")).toEqual(new Buffer("x")) + expect(blobStore.get("b", "invalidation-key-2")).toEqual(new Buffer("y")) diff --git a/spec/fixtures/native-cache/file-1.js b/spec/fixtures/native-cache/file-1.js new file mode 100644 index 000000000..ce195a18e --- /dev/null +++ b/spec/fixtures/native-cache/file-1.js @@ -0,0 +1 @@ +module.exports = function () { return 1; } diff --git a/spec/fixtures/native-cache/file-2.js b/spec/fixtures/native-cache/file-2.js new file mode 100644 index 000000000..e0cdf1485 --- /dev/null +++ b/spec/fixtures/native-cache/file-2.js @@ -0,0 +1 @@ +module.exports = function () { return 2; } diff --git a/spec/fixtures/native-cache/file-3.js b/spec/fixtures/native-cache/file-3.js new file mode 100644 index 000000000..36ca6e14a --- /dev/null +++ b/spec/fixtures/native-cache/file-3.js @@ -0,0 +1 @@ +module.exports = function () { return 3; } diff --git a/spec/fixtures/native-cache/file-4.js b/spec/fixtures/native-cache/file-4.js new file mode 100644 index 000000000..1b8fd4e15 --- /dev/null +++ b/spec/fixtures/native-cache/file-4.js @@ -0,0 +1 @@ +module.exports = function () { return "file-4" } diff --git a/spec/native-compile-cache-spec.coffee b/spec/native-compile-cache-spec.coffee new file mode 100644 index 000000000..1531deaf9 --- /dev/null +++ b/spec/native-compile-cache-spec.coffee @@ -0,0 +1,104 @@ +fs = require 'fs' +path = require 'path' +Module = require 'module' + +describe "NativeCompileCache", -> + nativeCompileCache = require '../src/native-compile-cache' + [fakeCacheStore, cachedFiles] = [] + + beforeEach -> + cachedFiles = [] + fakeCacheStore = jasmine.createSpyObj("cache store", ["set", "get", "has", "delete"]) + fakeCacheStore.has.andCallFake (cacheKey, invalidationKey) -> + fakeCacheStore.get(cacheKey, invalidationKey)? + fakeCacheStore.get.andCallFake (cacheKey, invalidationKey) -> + for entry in cachedFiles by -1 + continue if entry.cacheKey isnt cacheKey + continue if entry.invalidationKey isnt invalidationKey + return entry.cacheBuffer + return + fakeCacheStore.set.andCallFake (cacheKey, invalidationKey, cacheBuffer) -> + cachedFiles.push({cacheKey, invalidationKey, cacheBuffer}) + + nativeCompileCache.setCacheStore(fakeCacheStore) + nativeCompileCache.setV8Version("a-v8-version") + nativeCompileCache.install() + + it "writes and reads from the cache storage when requiring files", -> + fn1 = require('./fixtures/native-cache/file-1') + fn2 = require('./fixtures/native-cache/file-2') + + expect(cachedFiles.length).toBe(2) + + expect(cachedFiles[0].cacheKey).toBe(require.resolve('./fixtures/native-cache/file-1')) + expect(cachedFiles[0].cacheBuffer).toBeInstanceOf(Uint8Array) + expect(cachedFiles[0].cacheBuffer.length).toBeGreaterThan(0) + expect(fn1()).toBe(1) + + expect(cachedFiles[1].cacheKey).toBe(require.resolve('./fixtures/native-cache/file-2')) + expect(cachedFiles[1].cacheBuffer).toBeInstanceOf(Uint8Array) + expect(cachedFiles[1].cacheBuffer.length).toBeGreaterThan(0) + expect(fn2()).toBe(2) + + delete Module._cache[require.resolve('./fixtures/native-cache/file-1')] + fn1 = require('./fixtures/native-cache/file-1') + expect(cachedFiles.length).toBe(2) + expect(fn1()).toBe(1) + + describe "when v8 version changes", -> + it "updates the cache of previously required files", -> + nativeCompileCache.setV8Version("version-1") + fn4 = require('./fixtures/native-cache/file-4') + + expect(cachedFiles.length).toBe(1) + expect(cachedFiles[0].cacheKey).toBe(require.resolve('./fixtures/native-cache/file-4')) + expect(cachedFiles[0].cacheBuffer).toBeInstanceOf(Uint8Array) + expect(cachedFiles[0].cacheBuffer.length).toBeGreaterThan(0) + expect(fn4()).toBe("file-4") + + nativeCompileCache.setV8Version("version-2") + delete Module._cache[require.resolve('./fixtures/native-cache/file-4')] + fn4 = require('./fixtures/native-cache/file-4') + + expect(cachedFiles.length).toBe(2) + expect(cachedFiles[1].cacheKey).toBe(require.resolve('./fixtures/native-cache/file-4')) + expect(cachedFiles[1].invalidationKey).not.toBe(cachedFiles[0].invalidationKey) + expect(cachedFiles[1].cacheBuffer).toBeInstanceOf(Uint8Array) + expect(cachedFiles[1].cacheBuffer.length).toBeGreaterThan(0) + + describe "when a previously required and cached file changes", -> + beforeEach -> + fs.writeFileSync path.resolve(__dirname + '/fixtures/native-cache/file-5'), """ + module.exports = function () { return "file-5" } + """ + + afterEach -> + fs.unlinkSync path.resolve(__dirname + '/fixtures/native-cache/file-5') + + it "removes it from the store and re-inserts it with the new cache", -> + fn5 = require('./fixtures/native-cache/file-5') + + expect(cachedFiles.length).toBe(1) + expect(cachedFiles[0].cacheKey).toBe(require.resolve('./fixtures/native-cache/file-5')) + expect(cachedFiles[0].cacheBuffer).toBeInstanceOf(Uint8Array) + expect(cachedFiles[0].cacheBuffer.length).toBeGreaterThan(0) + expect(fn5()).toBe("file-5") + + delete Module._cache[require.resolve('./fixtures/native-cache/file-5')] + fs.appendFileSync(require.resolve('./fixtures/native-cache/file-5'), "\n\n") + fn5 = require('./fixtures/native-cache/file-5') + + expect(cachedFiles.length).toBe(2) + expect(cachedFiles[1].cacheKey).toBe(require.resolve('./fixtures/native-cache/file-5')) + expect(cachedFiles[1].invalidationKey).not.toBe(cachedFiles[0].invalidationKey) + expect(cachedFiles[1].cacheBuffer).toBeInstanceOf(Uint8Array) + expect(cachedFiles[1].cacheBuffer.length).toBeGreaterThan(0) + + it "deletes previously cached code when the cache is an invalid file", -> + fakeCacheStore.has.andReturn(true) + fakeCacheStore.get.andCallFake -> new Buffer("an invalid cache") + + fn3 = require('./fixtures/native-cache/file-3') + + expect(fakeCacheStore.delete).toHaveBeenCalledWith(require.resolve('./fixtures/native-cache/file-3')) + expect(fn3()).toBe(3) diff --git a/src/atom-environment.coffee b/src/atom-environment.coffee index dae45f7a2..2dfa736a2 100644 --- a/src/atom-environment.coffee +++ b/src/atom-environment.coffee @@ -131,7 +131,7 @@ class AtomEnvironment extends Model # Call .loadOrCreate instead constructor: (params={}) -> - {@applicationDelegate, @window, @document, @clipboard, @configDirPath, @enablePersistence, onlyLoadBaseStyleSheets} = params + {@applicationDelegate, @window, @document, @blobStore, @clipboard, @configDirPath, @enablePersistence, onlyLoadBaseStyleSheets} = params @unloaded = false @loadTime = null @@ -733,8 +733,13 @@ class AtomEnvironment extends Model @storeWindowBackground() @packages.deactivatePackages() + @saveBlobStoreSync() @unloaded = true + saveBlobStoreSync: -> + if @enablePersistence + @blobStore.save() + openInitialEmptyEditorIfNecessary: -> return unless @config.get('core.openEmptyEditorOnStart') if @getLoadSettings().initialPaths?.length is 0 and @workspace.getPaneItems().length is 0 diff --git a/src/file-system-blob-store.js b/src/file-system-blob-store.js new file mode 100644 index 000000000..828e23e94 --- /dev/null +++ b/src/file-system-blob-store.js @@ -0,0 +1,138 @@ +'use strict' + +const fs = require('fs-plus') +const path = require('path') + +module.exports = +class FileSystemBlobStore { + static load (directory) { + let instance = new FileSystemBlobStore(directory) + instance.load() + return instance + } + + constructor (directory) { + this.blobFilename = path.join(directory, 'BLOB') + this.blobMapFilename = path.join(directory, 'MAP') + this.invalidationKeysFilename = path.join(directory, 'INVKEYS') + this.lockFilename = path.join(directory, 'LOCK') + this.reset() + } + + reset () { + this.inMemoryBlobs = new Map() + this.invalidationKeys = {} + this.storedBlob = new Buffer(0) + this.storedBlobMap = {} + this.usedKeys = new Set() + } + + load () { + if (!fs.existsSync(this.blobMapFilename)) { + return + } + if (!fs.existsSync(this.blobFilename)) { + return + } + if (!fs.existsSync(this.invalidationKeysFilename)) { + return + } + + try { + this.storedBlob = fs.readFileSync(this.blobFilename) + this.storedBlobMap = JSON.parse(fs.readFileSync(this.blobMapFilename)) + this.invalidationKeys = JSON.parse(fs.readFileSync(this.invalidationKeysFilename)) + } catch (e) { + this.reset() + } + } + + save () { + let dump = this.getDump() + let blobToStore = Buffer.concat(dump[0]) + let mapToStore = JSON.stringify(dump[1]) + let invalidationKeysToStore = JSON.stringify(this.invalidationKeys) + + let acquiredLock = false + try { + fs.writeFileSync(this.lockFilename, 'LOCK', {flag: 'wx'}) + acquiredLock = true + + fs.writeFileSync(this.blobFilename, blobToStore) + fs.writeFileSync(this.blobMapFilename, mapToStore) + fs.writeFileSync(this.invalidationKeysFilename, invalidationKeysToStore) + } catch (error) { + // Swallow the exception silently only if we fail to acquire the lock. + if (error.code !== 'EEXIST') { + throw error + } + } finally { + if (acquiredLock) { + fs.unlinkSync(this.lockFilename) + } + } + } + + has (key, invalidationKey) { + let containsKey = this.inMemoryBlobs.has(key) || this.storedBlobMap.hasOwnProperty(key) + let isValid = this.invalidationKeys[key] === invalidationKey + return containsKey && isValid + } + + get (key, invalidationKey) { + if (this.has(key, invalidationKey)) { + this.usedKeys.add(key) + return this.getFromMemory(key) || this.getFromStorage(key) + } + } + + set (key, invalidationKey, buffer) { + this.usedKeys.add(key) + this.invalidationKeys[key] = invalidationKey + return this.inMemoryBlobs.set(key, buffer) + } + + delete (key) { + this.inMemoryBlobs.delete(key) + delete this.storedBlobMap[key] + } + + getFromMemory (key) { + return this.inMemoryBlobs.get(key) + } + + getFromStorage (key) { + if (!this.storedBlobMap[key]) { + return + } + + return this.storedBlob.slice.apply(this.storedBlob, this.storedBlobMap[key]) + } + + getDump () { + let buffers = [] + let blobMap = {} + let currentBufferStart = 0 + + function dump (key, getBufferByKey) { + let buffer = getBufferByKey(key) + buffers.push(buffer) + blobMap[key] = [currentBufferStart, currentBufferStart + buffer.length] + currentBufferStart += buffer.length + } + + for (let key of this.inMemoryBlobs.keys()) { + if (this.usedKeys.has(key)) { + dump(key, this.getFromMemory.bind(this)) + } + } + + for (let key of Object.keys(this.storedBlobMap)) { + if (!blobMap[key] && this.usedKeys.has(key)) { + dump(key, this.getFromStorage.bind(this)) + } + } + + return [buffers, blobMap] + } +} diff --git a/src/initialize-application-window.coffee b/src/initialize-application-window.coffee index b855936e0..13562fae2 100644 --- a/src/initialize-application-window.coffee +++ b/src/initialize-application-window.coffee @@ -3,6 +3,7 @@ ApplicationDelegate = require './application-delegate' Clipboard = require './clipboard' TextEditor = require './text-editor' TextEditorComponent = require './text-editor-component' +FileSystemBlobStore = require './file-system-blob-store' CompileCache = require './compile-cache' ModuleCache = require './module-cache' @@ -54,7 +55,7 @@ require('whitespace') require('wrap-guide') # Like sands through the hourglass, so are the days of our lives. -module.exports = -> +module.exports = ({blobStore}) -> {updateProcessEnv} = require('./update-process-env') path = require 'path' require './window' @@ -75,7 +76,7 @@ module.exports = -> TextEditor.setClipboard(clipboard) window.atom = new AtomEnvironment({ - window, document, clipboard, + window, document, clipboard, blobStore, applicationDelegate: new ApplicationDelegate, configDirPath: process.env.ATOM_HOME, enablePersistence: true, diff --git a/src/initialize-test-window.coffee b/src/initialize-test-window.coffee index 3649fea94..794db3174 100644 --- a/src/initialize-test-window.coffee +++ b/src/initialize-test-window.coffee @@ -5,7 +5,7 @@ cloneObject = (object) -> clone[key] = value for key, value of object clone -module.exports = -> +module.exports = ({blobStore}) -> startCrashReporter = require('./crash-reporter-start') {remote} = require 'electron' @@ -77,6 +77,7 @@ module.exports = -> buildAtomEnvironment = (params) -> params = cloneObject(params) params.clipboard = clipboard unless params.hasOwnProperty("clipboard") + params.blobStore = blobStore unless params.hasOwnProperty("blobStore") params.onlyLoadBaseStyleSheets = true unless params.hasOwnProperty("onlyLoadBaseStyleSheets") new AtomEnvironment(params) diff --git a/src/native-compile-cache.js b/src/native-compile-cache.js new file mode 100644 index 000000000..3f3d05991 --- /dev/null +++ b/src/native-compile-cache.js @@ -0,0 +1,116 @@ +const Module = require('module') +const path = require('path') +const cachedVm = require('cached-run-in-this-context') +const crypto = require('crypto') + +function computeHash (contents) { + return crypto.createHash('sha1').update(contents, 'utf8').digest('hex') +} + +class NativeCompileCache { + constructor () { + this.cacheStore = null + this.previousModuleCompile = null + } + + setCacheStore (store) { + this.cacheStore = store + } + + setV8Version (v8Version) { + this.v8Version = v8Version.toString() + } + + install () { + this.savePreviousModuleCompile() + this.overrideModuleCompile() + } + + uninstall () { + this.restorePreviousModuleCompile() + } + + savePreviousModuleCompile () { + this.previousModuleCompile = Module.prototype._compile + } + + overrideModuleCompile () { + let self = this + let resolvedArgv = null + // Here we override Node's module.js + // (https://github.com/atom/node/blob/atom/lib/module.js#L378), changing + // only the bits that affect compilation in order to use the cached one. + Module.prototype._compile = function (content, filename) { + let moduleSelf = this + // remove shebang + content = content.replace(/^#!.*/, '') + function require (path) { + return moduleSelf.require(path) + } + require.resolve = function (request) { + return Module._resolveFilename(request, moduleSelf) + } + require.main = process.mainModule + + // Enable support to add extra extension types + require.extensions = Module._extensions + require.cache = Module._cache + + let dirname = path.dirname(filename) + + // create wrapper function + let wrapper = Module.wrap(content) + + let cacheKey = filename + let invalidationKey = computeHash(wrapper + self.v8Version) + let compiledWrapper = null + if (self.cacheStore.has(cacheKey, invalidationKey)) { + let buffer = self.cacheStore.get(cacheKey, invalidationKey) + let compilationResult = cachedVm.runInThisContextCached(wrapper, filename, buffer) + compiledWrapper = compilationResult.result + if (compilationResult.wasRejected) { + self.cacheStore.delete(cacheKey) + } + } else { + let compilationResult + try { + compilationResult = cachedVm.runInThisContext(wrapper, filename) + } catch (err) { + console.error(`Error running script ${filename}`) + throw err + } + if (compilationResult.cacheBuffer) { + self.cacheStore.set(cacheKey, invalidationKey, compilationResult.cacheBuffer) + } + compiledWrapper = compilationResult.result + } + if (global.v8debug) { + if (!resolvedArgv) { + // we enter the repl if we're not given a filename argument. + if (process.argv[1]) { + resolvedArgv = Module._resolveFilename(process.argv[1], null) + } else { + resolvedArgv = 'repl' + } + } + + // Set breakpoint on module start + if (filename === resolvedArgv) { + // Installing this dummy debug event listener tells V8 to start + // the debugger. Without it, the setBreakPoint() fails with an + // 'illegal access' error. + global.v8debug.Debug.setListener(function () {}) + global.v8debug.Debug.setBreakPoint(compiledWrapper, 0, 0) + } + } + let args = [moduleSelf.exports, require, moduleSelf, filename, dirname, process, global] + return compiledWrapper.apply(moduleSelf.exports, args) + } + } + + restorePreviousModuleCompile () { + Module.prototype._compile = this.previousModuleCompile + } +} + +module.exports = new NativeCompileCache() diff --git a/static/index.js b/static/index.js index 5e5ddc347..2154f72b1 100644 --- a/static/index.js +++ b/static/index.js @@ -4,6 +4,7 @@ const Module = require('module') const getWindowLoadSettings = require('../src/get-window-load-settings') const entryPointDirPath = __dirname + let blobStore = null let useSnapshot = false window.onload = function () { @@ -46,6 +47,14 @@ snapshotResult.entryPointDirPath = __dirname } + const FileSystemBlobStore = useSnapshot ? snapshotResult.customRequire('../src/file-system-blob-store.js') : require('../src/file-system-blob-store') + blobStore = FileSystemBlobStore.load(path.join(process.env.ATOM_HOME, 'blob-store')) + + const NativeCompileCache = useSnapshot ? snapshotResult.customRequire('../src/native-compile-cache.js') : require('../src/native-compile-cache') + NativeCompileCache.setCacheStore(blobStore) + NativeCompileCache.setV8Version(process.versions.v8) + NativeCompileCache.install() + if (getWindowLoadSettings().profileStartup) { profileStartup(Date.now() - startTime) } else { @@ -88,7 +97,7 @@ const initScriptPath = path.relative(entryPointDirPath, getWindowLoadSettings().windowInitializationScript) const initialize = useSnapshot ? snapshotResult.customRequire(initScriptPath) : require(initScriptPath) - return initialize().then(function () { + return initialize({blobStore: blobStore}).then(function () { electron.ipcRenderer.send('window-command', 'window:loaded') }) }