diff --git a/package.json b/package.json index c0be8aa69..b2866c167 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "atom-keymap": "^6.2.0", "babel-core": "^5.8.21", "bootstrap": "^3.3.4", - "cached-run-in-this-context": "0.4.0", + "cached-run-in-this-context": "0.4.1", "clear-cut": "^2.0.1", "coffee-script": "1.8.0", "color": "^0.7.3", @@ -88,7 +88,7 @@ "dev-live-reload": "0.47.0", "encoding-selector": "0.21.0", "exception-reporting": "0.37.0", - "find-and-replace": "0.194.0", + "find-and-replace": "0.195.0", "fuzzy-finder": "0.93.0", "git-diff": "0.57.0", "go-to-line": "0.30.0", diff --git a/spec/file-system-blob-store-spec.coffee b/spec/file-system-blob-store-spec.coffee index c947259f6..c1cc29449 100644 --- a/spec/file-system-blob-store-spec.coffee +++ b/spec/file-system-blob-store-spec.coffee @@ -9,61 +9,71 @@ describe "FileSystemBlobStore", -> blobStore = FileSystemBlobStore.load(storageDirectory) it "is empty when the file doesn't exist", -> - expect(blobStore.get("foo")).toBeUndefined() - expect(blobStore.get("bar")).toBeUndefined() + expect(blobStore.get("foo", "invalidation-key-1")).toBeUndefined() + expect(blobStore.get("bar", "invalidation-key-2")).toBeUndefined() it "allows to read and write buffers from/to memory without persisting them", -> - blobStore.set("foo", new Buffer("foo")) - blobStore.set("bar", new Buffer("bar")) + blobStore.set("foo", "invalidation-key-1", new Buffer("foo")) + blobStore.set("bar", "invalidation-key-2", new Buffer("bar")) - expect(blobStore.get("foo")).toEqual(new Buffer("foo")) - expect(blobStore.get("bar")).toEqual(new Buffer("bar")) + expect(blobStore.get("foo", "invalidation-key-1")).toEqual(new Buffer("foo")) + expect(blobStore.get("bar", "invalidation-key-2")).toEqual(new Buffer("bar")) + + expect(blobStore.get("foo", "unexisting-key")).toBeUndefined() + expect(blobStore.get("bar", "unexisting-key")).toBeUndefined() it "persists buffers when saved and retrieves them on load, giving priority to in-memory ones", -> - blobStore.set("foo", new Buffer("foo")) - blobStore.set("bar", new Buffer("bar")) + blobStore.set("foo", "invalidation-key-1", new Buffer("foo")) + blobStore.set("bar", "invalidation-key-2", new Buffer("bar")) blobStore.save() blobStore = FileSystemBlobStore.load(storageDirectory) - expect(blobStore.get("foo")).toEqual(new Buffer("foo")) - expect(blobStore.get("bar")).toEqual(new Buffer("bar")) + expect(blobStore.get("foo", "invalidation-key-1")).toEqual(new Buffer("foo")) + expect(blobStore.get("bar", "invalidation-key-2")).toEqual(new Buffer("bar")) + expect(blobStore.get("foo", "unexisting-key")).toBeUndefined() + expect(blobStore.get("bar", "unexisting-key")).toBeUndefined() - blobStore.set("foo", new Buffer("changed")) + blobStore.set("foo", "new-key", new Buffer("changed")) - expect(blobStore.get("foo")).toEqual(new Buffer("changed")) + expect(blobStore.get("foo", "new-key")).toEqual(new Buffer("changed")) + expect(blobStore.get("foo", "invalidation-key-1")).toBeUndefined() it "persists both in-memory and previously stored buffers when saved", -> - blobStore.set("foo", new Buffer("foo")) - blobStore.set("bar", new Buffer("bar")) + blobStore.set("foo", "invalidation-key-1", new Buffer("foo")) + blobStore.set("bar", "invalidation-key-2", new Buffer("bar")) blobStore.save() blobStore = FileSystemBlobStore.load(storageDirectory) - blobStore.set("bar", new Buffer("changed")) - blobStore.set("qux", new Buffer("qux")) + blobStore.set("bar", "invalidation-key-3", new Buffer("changed")) + blobStore.set("qux", "invalidation-key-4", new Buffer("qux")) blobStore.save() blobStore = FileSystemBlobStore.load(storageDirectory) - expect(blobStore.get("foo")).toEqual(new Buffer("foo")) - expect(blobStore.get("bar")).toEqual(new Buffer("changed")) - expect(blobStore.get("qux")).toEqual(new Buffer("qux")) + expect(blobStore.get("foo", "invalidation-key-1")).toEqual(new Buffer("foo")) + expect(blobStore.get("bar", "invalidation-key-3")).toEqual(new Buffer("changed")) + expect(blobStore.get("qux", "invalidation-key-4")).toEqual(new Buffer("qux")) + expect(blobStore.get("foo", "unexisting-key")).toBeUndefined() + expect(blobStore.get("bar", "invalidation-key-2")).toBeUndefined() + expect(blobStore.get("qux", "unexisting-key")).toBeUndefined() it "allows to delete keys from both memory and stored buffers", -> - blobStore.set("a", new Buffer("a")) - blobStore.set("b", new Buffer("b")) + blobStore.set("a", "invalidation-key-1", new Buffer("a")) + blobStore.set("b", "invalidation-key-2", new Buffer("b")) blobStore.save() blobStore = FileSystemBlobStore.load(storageDirectory) - blobStore.set("b", new Buffer("b")) - blobStore.set("c", new Buffer("c")) + blobStore.set("b", "invalidation-key-3", new Buffer("b")) + blobStore.set("c", "invalidation-key-4", new Buffer("c")) blobStore.delete("b") blobStore.delete("c") blobStore.save() blobStore = FileSystemBlobStore.load(storageDirectory) - expect(blobStore.get("a")).toEqual(new Buffer("a")) - expect(blobStore.get("b")).toBeUndefined() - expect(blobStore.get("c")).toBeUndefined() + expect(blobStore.get("a", "invalidation-key-1")).toEqual(new Buffer("a")) + expect(blobStore.get("b", "invalidation-key-2")).toBeUndefined() + expect(blobStore.get("b", "invalidation-key-3")).toBeUndefined() + expect(blobStore.get("c", "invalidation-key-4")).toBeUndefined() diff --git a/spec/fixtures/native-cache/file-4.js b/spec/fixtures/native-cache/file-4.js new file mode 100644 index 000000000..1b8fd4e15 --- /dev/null +++ b/spec/fixtures/native-cache/file-4.js @@ -0,0 +1 @@ +module.exports = function () { return "file-4" } diff --git a/spec/native-compile-cache-spec.coffee b/spec/native-compile-cache-spec.coffee index dd720e84d..9d6cb89b4 100644 --- a/spec/native-compile-cache-spec.coffee +++ b/spec/native-compile-cache-spec.coffee @@ -1,3 +1,7 @@ +fs = require 'fs' +path = require 'path' +Module = require 'module' + describe "NativeCompileCache", -> nativeCompileCache = require '../src/native-compile-cache' [fakeCacheStore, cachedFiles] = [] @@ -5,39 +9,92 @@ describe "NativeCompileCache", -> beforeEach -> cachedFiles = [] fakeCacheStore = jasmine.createSpyObj("cache store", ["set", "get", "has", "delete"]) + fakeCacheStore.has.andCallFake (cacheKey, invalidationKey) -> + fakeCacheStore.get(cacheKey, invalidationKey)? + fakeCacheStore.get.andCallFake (cacheKey, invalidationKey) -> + for entry in cachedFiles by -1 + continue if entry.cacheKey isnt cacheKey + continue if entry.invalidationKey isnt invalidationKey + return entry.cacheBuffer + return + fakeCacheStore.set.andCallFake (cacheKey, invalidationKey, cacheBuffer) -> + cachedFiles.push({cacheKey, invalidationKey, cacheBuffer}) + nativeCompileCache.setCacheStore(fakeCacheStore) + nativeCompileCache.setV8Version("a-v8-version") nativeCompileCache.install() it "writes and reads from the cache storage when requiring files", -> - fakeCacheStore.has.andReturn(false) - fakeCacheStore.set.andCallFake (filename, cacheBuffer) -> - cachedFiles.push({filename, cacheBuffer}) - fn1 = require('./fixtures/native-cache/file-1') fn2 = require('./fixtures/native-cache/file-2') expect(cachedFiles.length).toBe(2) - expect(cachedFiles[0].filename).toBe(require.resolve('./fixtures/native-cache/file-1')) + expect(cachedFiles[0].cacheKey).toBe(require.resolve('./fixtures/native-cache/file-1')) expect(cachedFiles[0].cacheBuffer).toBeInstanceOf(Uint8Array) expect(cachedFiles[0].cacheBuffer.length).toBeGreaterThan(0) expect(fn1()).toBe(1) - expect(cachedFiles[1].filename).toBe(require.resolve('./fixtures/native-cache/file-2')) + expect(cachedFiles[1].cacheKey).toBe(require.resolve('./fixtures/native-cache/file-2')) expect(cachedFiles[1].cacheBuffer).toBeInstanceOf(Uint8Array) expect(cachedFiles[1].cacheBuffer.length).toBeGreaterThan(0) expect(fn2()).toBe(2) - fakeCacheStore.has.andReturn(true) - fakeCacheStore.get.andReturn(cachedFiles[0].cacheBuffer) - fakeCacheStore.set.reset() - + delete Module._cache[require.resolve('./fixtures/native-cache/file-1')] fn1 = require('./fixtures/native-cache/file-1') - - expect(fakeCacheStore.set).not.toHaveBeenCalled() + expect(cachedFiles.length).toBe(2) expect(fn1()).toBe(1) - it "deletes previously cached code when the cache is not valid", -> + describe "when v8 version changes", -> + it "updates the cache of previously required files", -> + nativeCompileCache.setV8Version("version-1") + fn4 = require('./fixtures/native-cache/file-4') + + expect(cachedFiles.length).toBe(1) + expect(cachedFiles[0].cacheKey).toBe(require.resolve('./fixtures/native-cache/file-4')) + expect(cachedFiles[0].cacheBuffer).toBeInstanceOf(Uint8Array) + expect(cachedFiles[0].cacheBuffer.length).toBeGreaterThan(0) + expect(fn4()).toBe("file-4") + + nativeCompileCache.setV8Version("version-2") + delete Module._cache[require.resolve('./fixtures/native-cache/file-4')] + fn4 = require('./fixtures/native-cache/file-4') + + expect(cachedFiles.length).toBe(2) + expect(cachedFiles[1].cacheKey).toBe(require.resolve('./fixtures/native-cache/file-4')) + expect(cachedFiles[1].invalidationKey).not.toBe(cachedFiles[0].invalidationKey) + expect(cachedFiles[1].cacheBuffer).toBeInstanceOf(Uint8Array) + expect(cachedFiles[1].cacheBuffer.length).toBeGreaterThan(0) + + describe "when a previously required and cached file changes", -> + beforeEach -> + fs.writeFileSync path.resolve('./spec/fixtures/native-cache/file-5'), """ + module.exports = function () { return "file-5" } + """ + + afterEach -> + fs.unlinkSync path.resolve('./spec/fixtures/native-cache/file-5') + + it "removes it from the store and re-inserts it with the new cache", -> + fn5 = require('./fixtures/native-cache/file-5') + + expect(cachedFiles.length).toBe(1) + expect(cachedFiles[0].cacheKey).toBe(require.resolve('./fixtures/native-cache/file-5')) + expect(cachedFiles[0].cacheBuffer).toBeInstanceOf(Uint8Array) + expect(cachedFiles[0].cacheBuffer.length).toBeGreaterThan(0) + expect(fn5()).toBe("file-5") + + delete Module._cache[require.resolve('./fixtures/native-cache/file-5')] + fs.appendFileSync(require.resolve('./fixtures/native-cache/file-5'), "\n\n") + fn5 = require('./fixtures/native-cache/file-5') + + expect(cachedFiles.length).toBe(2) + expect(cachedFiles[1].cacheKey).toBe(require.resolve('./fixtures/native-cache/file-5')) + expect(cachedFiles[1].invalidationKey).not.toBe(cachedFiles[0].invalidationKey) + expect(cachedFiles[1].cacheBuffer).toBeInstanceOf(Uint8Array) + expect(cachedFiles[1].cacheBuffer.length).toBeGreaterThan(0) + + it "deletes previously cached code when the cache is an invalid file", -> fakeCacheStore.has.andReturn(true) fakeCacheStore.get.andCallFake -> new Buffer("an invalid cache") diff --git a/src/file-system-blob-store.js b/src/file-system-blob-store.js index fc6bdddf3..e565a8857 100644 --- a/src/file-system-blob-store.js +++ b/src/file-system-blob-store.js @@ -13,8 +13,10 @@ class FileSystemBlobStore { constructor (directory) { this.inMemoryBlobs = new Map() + this.invalidationKeys = {} this.blobFilename = path.join(directory, 'BLOB') this.blobMapFilename = path.join(directory, 'MAP') + this.invalidationKeysFilename = path.join(directory, 'INVKEYS') this.lockFilename = path.join(directory, 'LOCK') this.storedBlob = new Buffer(0) this.storedBlobMap = {} @@ -27,14 +29,19 @@ class FileSystemBlobStore { if (!fs.existsSync(this.blobFilename)) { return } + if (!fs.existsSync(this.invalidationKeysFilename)) { + return + } this.storedBlob = fs.readFileSync(this.blobFilename) this.storedBlobMap = JSON.parse(fs.readFileSync(this.blobMapFilename)) + this.invalidationKeys = JSON.parse(fs.readFileSync(this.invalidationKeysFilename)) } save () { let dump = this.getDump() let blobToStore = Buffer.concat(dump[0]) let mapToStore = JSON.stringify(dump[1]) + let invalidationKeysToStore = JSON.stringify(this.invalidationKeys) let acquiredLock = false try { @@ -43,6 +50,7 @@ class FileSystemBlobStore { fs.writeFileSync(this.blobFilename, blobToStore) fs.writeFileSync(this.blobMapFilename, mapToStore) + fs.writeFileSync(this.invalidationKeysFilename, invalidationKeysToStore) } catch (error) { // Swallow the exception silently only if we fail to acquire the lock. if (error.code !== 'EEXIST') { @@ -55,15 +63,20 @@ class FileSystemBlobStore { } } - has (key) { - return this.inMemoryBlobs.hasOwnProperty(key) || this.storedBlobMap.hasOwnProperty(key) + has (key, invalidationKey) { + let containsKey = this.inMemoryBlobs.has(key) || this.storedBlobMap.hasOwnProperty(key) + let isValid = this.invalidationKeys[key] === invalidationKey + return containsKey && isValid } - get (key) { - return this.getFromMemory(key) || this.getFromStorage(key) + get (key, invalidationKey) { + if (this.has(key, invalidationKey)) { + return this.getFromMemory(key) || this.getFromStorage(key) + } } - set (key, buffer) { + set (key, invalidationKey, buffer) { + this.invalidationKeys[key] = invalidationKey return this.inMemoryBlobs.set(key, buffer) } diff --git a/src/native-compile-cache.js b/src/native-compile-cache.js index a930466a5..50fa71fc3 100644 --- a/src/native-compile-cache.js +++ b/src/native-compile-cache.js @@ -3,6 +3,11 @@ const Module = require('module') const path = require('path') const cachedVm = require('cached-run-in-this-context') +const crypto = require('crypto') + +function computeHash (contents) { + return crypto.createHash('sha1').update(contents, 'utf8').digest('hex') +} class NativeCompileCache { constructor () { @@ -14,6 +19,10 @@ class NativeCompileCache { this.cacheStore = store } + setV8Version (v8Version) { + this.v8Version = v8Version.toString() + } + install () { this.savePreviousModuleCompile() this.overrideModuleCompile() @@ -28,20 +37,20 @@ class NativeCompileCache { } overrideModuleCompile () { - let cacheStore = this.cacheStore + let self = this let resolvedArgv = null // Here we override Node's module.js // (https://github.com/atom/node/blob/atom/lib/module.js#L378), changing // only the bits that affect compilation in order to use the cached one. Module.prototype._compile = function (content, filename) { - let self = this + let moduleSelf = this // remove shebang content = content.replace(/^\#\!.*/, '') function require (path) { - return self.require(path) + return moduleSelf.require(path) } require.resolve = function (request) { - return Module._resolveFilename(request, self) + return Module._resolveFilename(request, moduleSelf) } require.main = process.mainModule @@ -54,18 +63,20 @@ class NativeCompileCache { // create wrapper function let wrapper = Module.wrap(content) + let cacheKey = filename + let invalidationKey = computeHash(wrapper + self.v8Version) let compiledWrapper = null - if (cacheStore.has(filename)) { - let buffer = cacheStore.get(filename) + if (self.cacheStore.has(cacheKey, invalidationKey)) { + let buffer = self.cacheStore.get(cacheKey, invalidationKey) let compilationResult = cachedVm.runInThisContextCached(wrapper, filename, buffer) compiledWrapper = compilationResult.result if (compilationResult.wasRejected) { - cacheStore.delete(filename) + self.cacheStore.delete(cacheKey) } } else { let compilationResult = cachedVm.runInThisContext(wrapper, filename) if (compilationResult.cacheBuffer) { - cacheStore.set(filename, compilationResult.cacheBuffer) + self.cacheStore.set(cacheKey, invalidationKey, compilationResult.cacheBuffer) } compiledWrapper = compilationResult.result } @@ -88,8 +99,8 @@ class NativeCompileCache { global.v8debug.Debug.setBreakPoint(compiledWrapper, 0, 0) } } - let args = [self.exports, require, self, filename, dirname, process, global] - return compiledWrapper.apply(self.exports, args) + let args = [moduleSelf.exports, require, moduleSelf, filename, dirname, process, global] + return compiledWrapper.apply(moduleSelf.exports, args) } } diff --git a/static/index.js b/static/index.js index 2a5bcad3a..8a7761198 100644 --- a/static/index.js +++ b/static/index.js @@ -19,6 +19,7 @@ path.join(process.env.ATOM_HOME, 'blob-store/') ) NativeCompileCache.setCacheStore(blobStore) + NativeCompileCache.setV8Version(process.versions.v8) NativeCompileCache.install() // Normalize to make sure drive letter case is consistent on Windows