Merge pull request #3761 from atom/ks-require-cache

Cache requires across installed packages

CONTRIBUTING.md

@@ -62,6 +62,8 @@ For more information on how to work with Atom's official packages, see
     * Use `path.join()` to concatenate filenames.
     * Use `os.tmpdir()` rather than `/tmp` when you need to reference the
       temporary directory.
+* Using a plain `return` when returning explicitly at the end of a function.
+    * Not `return null`, `return undefined`, `null`, or `undefined`
 
 ## Git Commit Messages
 

build/tasks/build-task.coffee

@@ -153,7 +153,7 @@ module.exports = (grunt) ->
       fs.writeFileSync path.join(appDir, 'node_modules', 'symbols-view', 'vendor', 'ctags-win32.exe.ignore'), ''
       fs.writeFileSync path.join(shellAppDir, 'atom.exe.gui'), ''
 
-    dependencies = ['compile', "generate-license:save"]
+    dependencies = ['compile', 'generate-license:save', 'generate-module-cache']
     dependencies.push('copy-info-plist') if process.platform is 'darwin'
     dependencies.push('set-exe-icon') if process.platform is 'win32'
     grunt.task.run(dependencies...)

build/tasks/generate-module-cache-task.coffee

@@ -0,0 +1,58 @@
+path = require 'path'
+fs = require 'fs-plus'
+ModuleCache = require '../../src/module-cache'
+
+module.exports = (grunt) ->
+  grunt.registerTask 'generate-module-cache', 'Generate a module cache for all core modules and packages', ->
+    appDir = grunt.config.get('atom.appDir')
+
+    {packageDependencies} = grunt.file.readJSON('package.json')
+
+    for packageName, version of packageDependencies
+      ModuleCache.create(path.join(appDir, 'node_modules', packageName))
+
+    ModuleCache.create(appDir)
+
+    metadata = grunt.file.readJSON(path.join(appDir, 'package.json'))
+
+    metadata._atomModuleCache.folders.forEach (folder) ->
+      if '' in folder.paths
+        folder.paths = [
+          ''
+          'exports'
+          'spec'
+          'src'
+          'src/browser'
+          'static'
+          'vendor'
+        ]
+
+    # Reactionary does not have an explicit react dependency
+    metadata._atomModuleCache.folders.push
+      paths: [
+        'node_modules/reactionary-atom-fork/lib'
+      ]
+      dependencies: {
+        'react-atom-fork': metadata.dependencies['react-atom-fork']
+      }
+
+    validExtensions = ['.js', '.coffee', '.json', '.node']
+
+    extensions = {}
+    onFile = (filePath) ->
+      filePath = path.relative(appDir, filePath)
+      segments = filePath.split(path.sep)
+      return if segments.length > 1 and not (segments[0] in ['exports', 'node_modules', 'src', 'static', 'vendor'])
+
+      extension = path.extname(filePath)
+      if extension in validExtensions
+        extensions[extension] ?= []
+        extensions[extension].push(filePath)
+
+    onDirectory = -> true
+
+    files = fs.traverseTreeSync(appDir, onFile, onDirectory)
+
+    metadata._atomModuleCache.extensions = extensions
+
+    grunt.file.write(path.join(appDir, 'package.json'), JSON.stringify(metadata, null, 2))

package.json

@@ -54,11 +54,11 @@
     "scoped-property-store": "^0.13.2",
     "scrollbar-style": "^1.0.2",
     "season": "^1.0.2",
-    "semver": "1.1.4",
+    "semver": "2.2.1",
     "serializable": "^1",
     "space-pen": "3.8.0",
     "temp": "0.7.0",
-    "text-buffer": "^3.2.8",
+    "text-buffer": "^3.2.9",
     "theorist": "^1.0.2",
     "underscore-plus": "^1.5.1",
     "vm-compatibility-layer": "0.1.0"

spec/fixtures/module-cache/file.json

@@ -0,0 +1,3 @@
+{
+  "foo": "bar"
+}

spec/module-cache-spec.coffee

@@ -0,0 +1,92 @@
+path = require 'path'
+Module = require 'module'
+fs = require 'fs-plus'
+temp = require 'temp'
+ModuleCache = require '../src/module-cache'
+
+describe 'ModuleCache', ->
+  beforeEach ->
+    spyOn(Module, '_findPath').andCallThrough()
+
+  it 'resolves atom shell module paths without hitting the filesystem', ->
+    builtins = ModuleCache.cache.builtins
+    expect(Object.keys(builtins).length).toBeGreaterThan 0
+
+    for builtinName, builtinPath of builtins
+      expect(require.resolve(builtinName)).toBe builtinPath
+      expect(fs.isFileSync(require.resolve(builtinName)))
+
+    expect(Module._findPath.callCount).toBe 0
+
+  it 'resolves relative core paths without hitting the filesystem', ->
+    ModuleCache.add atom.getLoadSettings().resourcePath, {
+      _atomModuleCache:
+        extensions:
+          '.json': [
+            path.join('spec', 'fixtures', 'module-cache', 'file.json')
+          ]
+    }
+    expect(require('./fixtures/module-cache/file.json').foo).toBe 'bar'
+    expect(Module._findPath.callCount).toBe 0
+
+  it 'resolves module paths when a compatible version is provided by core', ->
+    packagePath = fs.realpathSync(temp.mkdirSync('atom-package'))
+    ModuleCache.add packagePath, {
+      _atomModuleCache:
+        folders: [{
+          paths: [
+            ''
+          ]
+          dependencies:
+            'underscore-plus': '*'
+        }]
+    }
+    ModuleCache.add atom.getLoadSettings().resourcePath, {
+      _atomModuleCache:
+        dependencies: [{
+          name: 'underscore-plus'
+          version: require('underscore-plus/package.json').version
+          path: path.join('node_modules', 'underscore-plus', 'lib', 'underscore-plus.js')
+        }]
+    }
+
+    indexPath = path.join(packagePath, 'index.js')
+    fs.writeFileSync indexPath, """
+      exports.load = function() { require('underscore-plus'); };
+    """
+
+    packageMain = require(indexPath)
+    Module._findPath.reset()
+    packageMain.load()
+    expect(Module._findPath.callCount).toBe 0
+
+  it 'does not resolve module paths when no compatible version is provided by core', ->
+    packagePath = fs.realpathSync(temp.mkdirSync('atom-package'))
+    ModuleCache.add packagePath, {
+      _atomModuleCache:
+        folders: [{
+          paths: [
+            ''
+          ]
+          dependencies:
+            'underscore-plus': '0.0.1'
+        }]
+    }
+    ModuleCache.add atom.getLoadSettings().resourcePath, {
+      _atomModuleCache:
+        dependencies: [{
+          name: 'underscore-plus'
+          version: require('underscore-plus/package.json').version
+          path: path.join('node_modules', 'underscore-plus', 'lib', 'underscore-plus.js')
+        }]
+    }
+
+    indexPath = path.join(packagePath, 'index.js')
+    fs.writeFileSync indexPath, """
+      exports.load = function() { require('underscore-plus'); };
+    """
+
+    packageMain = require(indexPath)
+    Module._findPath.reset()
+    expect(-> packageMain.load()).toThrow()
+    expect(Module._findPath.callCount).toBe 1

src/coffee-cache.coffee

@@ -45,3 +45,11 @@ module.exports =
       writable: false
       value: requireCoffeeScript
     })
+  addPathToCache: (filePath) ->
+    extension = path.extname(filePath)
+    if extension is '.coffee'
+      content = fs.readFileSync(filePath, 'utf8')
+      cachePath = getCachePath(coffee)
+      compileCoffeeScript(coffee, filePath, cachePath)
+    else if extension is '.cson'
+      CSON.readFileSync(filePath)

src/module-cache.coffee

@@ -0,0 +1,270 @@
+Module = require 'module'
+path = require 'path'
+semver = require 'semver'
+
+# Extend semver.Range to memoize matched versions for speed
+class Range extends semver.Range
+  constructor: ->
+    super
+    @matchedVersions = new Set()
+
+  test: (version) ->
+    return true if @matchedVersions.has(version)
+
+    matches = super
+    @matchedVersions.add(version) if matches
+    matches
+
+nativeModules = process.binding('natives')
+
+cache =
+  builtins: {}
+  debug: false
+  dependencies: {}
+  extensions: {}
+  folders: {}
+  ranges: {}
+  registered: false
+  resourcePath: null
+  resourcePathWithTrailingSlash: null
+
+# isAbsolute is inlined from fs-plus so that fs-plus itself can be required
+# from this cache.
+if process.platform is 'win32'
+  isAbsolute = (pathToCheck) ->
+    pathToCheck and (pathToCheck[1] is ':' or (pathToCheck[0] is '\\' and pathToCheck[1] is '\\'))
+else
+  isAbsolute = (pathToCheck) ->
+    pathToCheck and pathToCheck[0] is '/'
+
+isCorePath = (pathToCheck) ->
+  pathToCheck.startsWith(cache.resourcePathWithTrailingSlash)
+
+loadDependencies = (modulePath, rootPath, rootMetadata, moduleCache) ->
+  fs = require 'fs-plus'
+
+  for childPath in fs.listSync(path.join(modulePath, 'node_modules'))
+    continue if path.basename(childPath) is '.bin'
+    continue if rootPath is modulePath and rootMetadata.packageDependencies?.hasOwnProperty(path.basename(childPath))
+
+    childMetadataPath = path.join(childPath, 'package.json')
+    continue unless fs.isFileSync(childMetadataPath)
+
+    childMetadata = JSON.parse(fs.readFileSync(childMetadataPath))
+    if childMetadata?.version
+      try
+        mainPath = require.resolve(childPath)
+
+      if mainPath
+        moduleCache.dependencies.push
+          name: childMetadata.name
+          version: childMetadata.version
+          path: path.relative(rootPath, mainPath)
+
+      loadDependencies(childPath, rootPath, rootMetadata, moduleCache)
+
+  return
+
+loadFolderCompatibility = (modulePath, rootPath, rootMetadata, moduleCache) ->
+  fs = require 'fs-plus'
+
+  metadataPath = path.join(modulePath, 'package.json')
+  return unless fs.isFileSync(metadataPath)
+
+  dependencies = JSON.parse(fs.readFileSync(metadataPath))?.dependencies ? {}
+
+  for name, version of dependencies
+    try
+      new semver.Range(version)
+    catch error
+      delete dependencies[name]
+
+  onDirectory = (childPath) ->
+    path.basename(childPath) isnt 'node_modules'
+
+  extensions = ['.js', '.coffee', '.json', '.node']
+  paths = {}
+  onFile = (childPath) ->
+    if path.extname(childPath) in extensions
+      relativePath = path.relative(rootPath, path.dirname(childPath))
+      paths[relativePath] = true
+  fs.traverseTreeSync(modulePath, onFile, onDirectory)
+
+  paths = Object.keys(paths)
+  if paths.length > 0 and Object.keys(dependencies).length > 0
+    moduleCache.folders.push({paths, dependencies})
+
+  for childPath in fs.listSync(path.join(modulePath, 'node_modules'))
+    continue if path.basename(childPath) is '.bin'
+    continue if rootPath is modulePath and rootMetadata.packageDependencies?.hasOwnProperty(path.basename(childPath))
+
+    loadFolderCompatibility(childPath, rootPath, rootMetadata, moduleCache)
+
+  return
+
+satisfies = (version, rawRange) ->
+  unless parsedRange = cache.ranges[rawRange]
+    parsedRange = new Range(rawRange)
+    cache.ranges[rawRange] = parsedRange
+  parsedRange.test(version)
+
+resolveFilePath = (relativePath, parentModule) ->
+  return unless relativePath
+  return unless parentModule?.filename
+  return unless relativePath[0] is '.' or isAbsolute(relativePath)
+
+  resolvedPath = path.resolve(path.dirname(parentModule.filename), relativePath)
+  return unless isCorePath(resolvedPath)
+
+  extension = path.extname(resolvedPath)
+  if extension
+    return resolvedPath if cache.extensions[extension]?.has(resolvedPath)
+  else
+    for extension, paths of cache.extensions
+      resolvedPathWithExtension = "#{resolvedPath}#{extension}"
+      return resolvedPathWithExtension if paths.has(resolvedPathWithExtension)
+
+  return
+
+resolveModulePath = (relativePath, parentModule) ->
+  return unless relativePath
+  return unless parentModule?.filename
+
+  return if nativeModules.hasOwnProperty(relativePath)
+  return if relativePath[0] is '.'
+  return if isAbsolute(relativePath)
+
+  folderPath = path.dirname(parentModule.filename)
+
+  range = cache.folders[folderPath]?[relativePath]
+  unless range?
+    if builtinPath = cache.builtins[relativePath]
+      return builtinPath
+    else
+      return
+
+  candidates = cache.dependencies[relativePath]
+  return unless candidates?
+
+  for version, resolvedPath of candidates
+    if Module._cache.hasOwnProperty(resolvedPath) or isCorePath(resolvedPath)
+      return resolvedPath if satisfies(version, range)
+
+  return
+
+registerBuiltins = (devMode) ->
+  if devMode
+    cache.builtins.atom = path.join(cache.resourcePath, 'exports', 'atom.coffee')
+  else
+    cache.builtins.atom = path.join(cache.resourcePath, 'exports', 'atom.js')
+
+  atomShellRoot = path.join(process.resourcesPath, 'atom')
+
+  commonRoot = path.join(atomShellRoot, 'common', 'api', 'lib')
+  commonBuiltins = ['callbacks-registry', 'clipboard', 'crash-reporter', 'screen', 'shell']
+  for builtin in commonBuiltins
+    cache.builtins[builtin] = path.join(commonRoot, "#{builtin}.js")
+
+  rendererRoot = path.join(atomShellRoot, 'renderer', 'api', 'lib')
+  rendererBuiltins = ['ipc', 'remote']
+  for builtin in rendererBuiltins
+    cache.builtins[builtin] = path.join(rendererRoot, "#{builtin}.js")
+
+if cache.debug
+  cache.findPathCount = 0
+  cache.findPathTime = 0
+  cache.loadCount = 0
+  cache.requireTime = 0
+  global.moduleCache = cache
+
+  originalLoad = Module::load
+  Module::load = ->
+    cache.loadCount++
+    originalLoad.apply(this, arguments)
+
+  originalRequire = Module::require
+  Module::require = ->
+    startTime = Date.now()
+    exports = originalRequire.apply(this, arguments)
+    cache.requireTime += Date.now() - startTime
+    exports
+
+  originalFindPath = Module._findPath
+  Module._findPath = (request, paths) ->
+    cacheKey = JSON.stringify({request, paths})
+    cache.findPathCount++ unless Module._pathCache[cacheKey]
+
+    startTime = Date.now()
+    foundPath = originalFindPath.apply(global, arguments)
+    cache.findPathTime += Date.now() - startTime
+    foundPath
+
+exports.create = (modulePath) ->
+  fs = require 'fs-plus'
+
+  modulePath = fs.realpathSync(modulePath)
+  metadataPath = path.join(modulePath, 'package.json')
+  metadata = JSON.parse(fs.readFileSync(metadataPath))
+
+  moduleCache =
+    version: 1
+    dependencies: []
+    folders: []
+
+  loadDependencies(modulePath, modulePath, metadata, moduleCache)
+  loadFolderCompatibility(modulePath, modulePath, metadata, moduleCache)
+
+  metadata._atomModuleCache = moduleCache
+  fs.writeFileSync(metadataPath, JSON.stringify(metadata, null, 2))
+
+  return
+
+exports.register = ({resourcePath, devMode}={}) ->
+  return if cache.registered
+
+  originalResolveFilename = Module._resolveFilename
+  Module._resolveFilename = (relativePath, parentModule) ->
+    resolvedPath = resolveModulePath(relativePath, parentModule)
+    resolvedPath ?= resolveFilePath(relativePath, parentModule)
+    resolvedPath ? originalResolveFilename(relativePath, parentModule)
+
+  cache.registered = true
+  cache.resourcePath = resourcePath
+  cache.resourcePathWithTrailingSlash = "#{resourcePath}#{path.sep}"
+  registerBuiltins(devMode)
+
+  return
+
+exports.add = (directoryPath, metadata) ->
+  # path.join isn't used in this function for speed since path.join calls
+  # path.normalize and all the paths are already normalized here.
+
+  unless metadata?
+    try
+      metadata = require("#{directoryPath}#{path.sep}package.json")
+    catch error
+      return
+
+  cacheToAdd = metadata?._atomModuleCache
+  return unless cacheToAdd?
+
+  for dependency in cacheToAdd.dependencies ? []
+    cache.dependencies[dependency.name] ?= {}
+    cache.dependencies[dependency.name][dependency.version] ?= "#{directoryPath}#{path.sep}#{dependency.path}"
+
+  for entry in cacheToAdd.folders ? []
+    for folderPath in entry.paths
+      if folderPath
+        cache.folders["#{directoryPath}#{path.sep}#{folderPath}"] = entry.dependencies
+      else
+        cache.folders[directoryPath] = entry.dependencies
+
+  if directoryPath is cache.resourcePath
+    for extension, paths of cacheToAdd.extensions
+      cache.extensions[extension] ?= new Set()
+      for filePath in paths
+        cache.extensions[extension].add("#{directoryPath}#{path.sep}#{filePath}")
+
+  return
+
+exports.cache = cache

src/package.coffee

@@ -10,6 +10,7 @@ Q = require 'q'
 {deprecate} = require 'grim'
 
 $ = null # Defer require in case this is in the window-less browser process
+ModuleCache = require './module-cache'
 ScopedProperties = require './scoped-properties'
 
 # Loads and activates a package's main module and resources such as
@@ -47,6 +48,7 @@ class Package
     @emitter = new Emitter
     @metadata ?= Package.loadMetadata(@path)
     @name = @metadata?.name ? path.basename(@path)
+    ModuleCache.add(@path, @metadata)
     @reset()
 
   ###

src/window-bootstrap.coffee

@@ -1,14 +1,10 @@
 # Like sands through the hourglass, so are the days of our lives.
-startTime = Date.now()
-
 require './window'
 
 Atom = require './atom'
 window.atom = Atom.loadOrCreate('editor')
 atom.initialize()
 atom.startEditorWindow()
-window.atom.loadTime = Date.now() - startTime
-console.log "Window load time: #{atom.getWindowLoadTime()}ms"
 
 # Workaround for focus getting cleared upon window creation
 windowFocused = ->

static/index.js

@@ -1,10 +1,19 @@
 window.onload = function() {
-  var path = require('path');
-  var ipc = require('ipc');
   try {
+    var startTime = Date.now();
+
     // Skip "?loadSettings=".
     var loadSettings = JSON.parse(decodeURIComponent(location.search.substr(14)));
 
+    // Require before the module cache in dev mode
+    if (loadSettings.devMode) {
+      require('coffee-script').register();
+    }
+
+    ModuleCache = require('../src/module-cache');
+    ModuleCache.register(loadSettings);
+    ModuleCache.add(loadSettings.resourcePath);
+
     // Start the crash reporter before anything else.
     require('crash-reporter').start({
       productName: 'Atom',
@@ -15,10 +24,20 @@ window.onload = function() {
     });
 
     require('vm-compatibility-layer');
-    require('coffee-script').register();
-    require(path.resolve(__dirname, '..', 'src', 'coffee-cache')).register();
+
+    if (!loadSettings.devMode) {
+      require('coffee-script').register();
+    }
+
+    require('../src/coffee-cache').register();
+
     require(loadSettings.bootstrapScript);
-    ipc.sendChannel('window-command', 'window:loaded')
+    require('ipc').sendChannel('window-command', 'window:loaded');
+
+    if (global.atom) {
+      global.atom.loadTime = Date.now() - startTime;
+      console.log('Window load time: ' + global.atom.getWindowLoadTime() + 'ms');
+    }
   }
   catch (error) {
     var currentWindow = require('remote').getCurrentWindow();