Merge pull request #13101 from atom/mkt-package-specific-transpilation

Per-package transpilation

package.json

@@ -42,6 +42,7 @@
     "less-cache": "0.23",
     "line-top-index": "0.2.0",
     "marked": "^0.3.6",
+    "minimatch": "^3.0.3",
     "mocha": "2.5.1",
     "normalize-package-data": "^2.0.0",
     "nslog": "^3",

spec/package-transpilation-registry-spec.js

@@ -0,0 +1,145 @@
+/** @babel */
+import fs from 'fs'
+import path from 'path'
+
+import {it, fit, ffit, fffit, beforeEach, afterEach} from './async-spec-helpers'
+
+import PackageTranspilationRegistry from '../src/package-transpilation-registry'
+
+const originalCompiler = {
+  getCachePath: (sourceCode, filePath) => {
+    return "orig-cache-path"
+  },
+
+  compile: (sourceCode, filePath) => {
+    return sourceCode + "-original-compiler"
+  },
+
+  shouldCompile: (sourceCode, filePath) => {
+    return path.extname(filePath) === '.js'
+  }
+}
+
+describe("PackageTranspilationRegistry", () => {
+  let registry
+  let wrappedCompiler
+
+  beforeEach(() => {
+    registry = new PackageTranspilationRegistry()
+    wrappedCompiler = registry.wrapTranspiler(originalCompiler)
+  })
+
+  it('falls through to the original compiler by default', () => {
+    spyOn(originalCompiler, 'getCachePath')
+    spyOn(originalCompiler, 'compile')
+    spyOn(originalCompiler, 'shouldCompile')
+
+    wrappedCompiler.getCachePath('source', '/path/to/file.js')
+    wrappedCompiler.compile('source', '/path/to/filejs')
+    wrappedCompiler.shouldCompile('source', '/path/to/file.js')
+
+    expect(originalCompiler.getCachePath).toHaveBeenCalled()
+    expect(originalCompiler.compile).toHaveBeenCalled()
+    expect(originalCompiler.shouldCompile).toHaveBeenCalled()
+  })
+
+  describe('when a file is contained in a path that has custom transpilation', () => {
+    const hitPath = '/path/to/lib/file.js'
+    const hitPathCoffee = '/path/to/file2.coffee'
+    const missPath = '/path/other/file3.js'
+    const hitPathMissSubdir = '/path/to/file4.js'
+    const hitPathMissExt = '/path/to/file5.ts'
+    const nodeModulesFolder = '/path/to/lib/node_modules/file6.js'
+    const hitNonStandardExt = '/path/to/file7.omgwhatisthis'
+
+    const jsSpec = { glob: "lib/**/*.js", transpiler: './transpiler-js', options: { type: 'js' } }
+    const coffeeSpec = { glob: "*.coffee", transpiler: './transpiler-coffee', options: { type: 'coffee' } }
+    const omgSpec = { glob: "*.omgwhatisthis", transpiler: './transpiler-omg', options: { type: 'omg' } }
+
+    const jsTranspiler = {
+      transpile: (sourceCode, filePath, options) => {
+        return {code: sourceCode + "-transpiler-js"}
+      },
+
+      getCacheKeyData: (sourceCode, filePath, options) => {
+        return 'js-transpiler-cache-data'
+      }
+    }
+
+    const coffeeTranspiler = {
+      transpile: (sourceCode, filePath, options) => {
+        return {code: sourceCode + "-transpiler-coffee"}
+      },
+
+      getCacheKeyData: (sourceCode, filePath, options) => {
+        return 'coffee-transpiler-cache-data'
+      }
+    }
+
+    const omgTranspiler = {
+      transpile: (sourceCode, filePath, options) => {
+        return {code: sourceCode + "-transpiler-omg"}
+      },
+
+      getCacheKeyData: (sourceCode, filePath, options) => {
+        return 'omg-transpiler-cache-data'
+      }
+    }
+
+    beforeEach(() => {
+      jsSpec._transpilerSource = "js-transpiler-source"
+      coffeeSpec._transpilerSource = "coffee-transpiler-source"
+      omgTranspiler._transpilerSource = "omg-transpiler-source"
+
+      spyOn(registry, "getTranspiler").andCallFake(spec => {
+        if (spec.transpiler === './transpiler-js') return jsTranspiler
+        if (spec.transpiler === './transpiler-coffee') return coffeeTranspiler
+        if (spec.transpiler === './transpiler-omg') return omgTranspiler
+        throw new Error('bad transpiler path ' + spec.transpiler)
+      })
+
+      registry.addTranspilerConfigForPath('/path/to', 'my-package', [
+        jsSpec, coffeeSpec, omgSpec
+      ])
+    })
+
+    it('always returns true from shouldCompile for a file in that dir that match a glob', () => {
+      spyOn(originalCompiler, 'shouldCompile').andReturn(false)
+      expect(wrappedCompiler.shouldCompile('source', hitPath)).toBe(true)
+      expect(wrappedCompiler.shouldCompile('source', hitPathCoffee)).toBe(true)
+      expect(wrappedCompiler.shouldCompile('source', hitNonStandardExt)).toBe(true)
+      expect(wrappedCompiler.shouldCompile('source', hitPathMissExt)).toBe(false)
+      expect(wrappedCompiler.shouldCompile('source', hitPathMissSubdir)).toBe(false)
+      expect(wrappedCompiler.shouldCompile('source', missPath)).toBe(false)
+      expect(wrappedCompiler.shouldCompile('source', nodeModulesFolder)).toBe(false)
+    })
+
+    it('calls getCacheKeyData on the transpiler to get additional cache key data', () => {
+      spyOn(registry, "getTranspilerPath").andReturn("./transpiler-js")
+      spyOn(jsTranspiler, 'getCacheKeyData').andCallThrough()
+
+      wrappedCompiler.getCachePath('source', missPath, jsSpec)
+      expect(jsTranspiler.getCacheKeyData).not.toHaveBeenCalled()
+      wrappedCompiler.getCachePath('source', hitPath, jsSpec)
+      expect(jsTranspiler.getCacheKeyData).toHaveBeenCalled()
+    })
+
+    it('compiles files matching a glob with the associated transpiler, and the old one otherwise', () => {
+      spyOn(jsTranspiler, "transpile").andCallThrough()
+      spyOn(coffeeTranspiler, "transpile").andCallThrough()
+      spyOn(omgTranspiler, "transpile").andCallThrough()
+
+      expect(wrappedCompiler.compile('source', hitPath)).toEqual('source-transpiler-js')
+      expect(jsTranspiler.transpile).toHaveBeenCalledWith('source', hitPath, jsSpec.options)
+      expect(wrappedCompiler.compile('source', hitPathCoffee)).toEqual('source-transpiler-coffee')
+      expect(coffeeTranspiler.transpile).toHaveBeenCalledWith('source', hitPathCoffee, coffeeSpec.options)
+      expect(wrappedCompiler.compile('source', hitNonStandardExt)).toEqual('source-transpiler-omg')
+      expect(omgTranspiler.transpile).toHaveBeenCalledWith('source', hitNonStandardExt, omgSpec.options)
+
+      expect(wrappedCompiler.compile('source', missPath)).toEqual('source-original-compiler')
+      expect(wrappedCompiler.compile('source', hitPathMissExt)).toEqual('source-original-compiler')
+      expect(wrappedCompiler.compile('source', hitPathMissSubdir)).toEqual('source-original-compiler')
+      expect(wrappedCompiler.compile('source', nodeModulesFolder)).toEqual('source-original-compiler')
+    })
+  })
+})

src/compile-cache.js

@@ -7,12 +7,26 @@
 
 var path = require('path')
 var fs = require('fs-plus')
+
+var PackageTranspilationRegistry = require('./package-transpilation-registry')
 var CSON = null
 
+var packageTranspilationRegistry = new PackageTranspilationRegistry()
+
 var COMPILERS = {
-  '.js': require('./babel'),
-  '.ts': require('./typescript'),
-  '.coffee': require('./coffee-script')
+  '.js': packageTranspilationRegistry.wrapTranspiler(require('./babel')),
+  '.ts': packageTranspilationRegistry.wrapTranspiler(require('./typescript')),
+  '.coffee': packageTranspilationRegistry.wrapTranspiler(require('./coffee-script'))
+}
+
+exports.addTranspilerConfigForPath = function (packagePath, packageName, config) {
+  packagePath = fs.realpathSync(packagePath)
+  packageTranspilationRegistry.addTranspilerConfigForPath(packagePath, packageName, config)
+}
+
+exports.removeTranspilerConfigForPath = function (packagePath) {
+  packagePath = fs.realpathSync(packagePath)
+  packageTranspilationRegistry.removeTranspilerConfigForPath(packagePath)
 }
 
 var cacheStats = {}

src/package-transpilation-registry.js

@@ -0,0 +1,161 @@
+'use strict'
+// This file is required by compile-cache, which is required directly from
+// apm, so it can only use the subset of newer JavaScript features that apm's
+// version of Node supports. Strict mode is required for block scoped declarations.
+
+const crypto = require('crypto')
+const fs = require('fs')
+const path = require('path')
+
+const minimatch = require('minimatch')
+
+let Resolve = null
+
+class PackageTranspilationRegistry {
+  constructor () {
+    this.configByPackagePath = {}
+    this.specByFilePath = {}
+    this.transpilerPaths = {}
+  }
+
+  addTranspilerConfigForPath (packagePath, packageName, config) {
+    this.configByPackagePath[packagePath] = {
+      name: packageName,
+      path: packagePath,
+      specs: config
+    }
+  }
+
+  removeTranspilerConfigForPath (packagePath) {
+    delete this.configByPackagePath[packagePath]
+  }
+
+  // Wraps the transpiler in an object with the same interface
+  // that falls back to the original transpiler implementation if and
+  // only if a package hasn't registered its desire to transpile its own source.
+  wrapTranspiler (transpiler) {
+    return {
+      getCachePath: (sourceCode, filePath) => {
+        const spec = this.getPackageTranspilerSpecForFilePath(filePath)
+        if (spec) {
+          return this.getCachePath(sourceCode, filePath, spec)
+        }
+
+        return transpiler.getCachePath(sourceCode, filePath)
+      },
+
+      compile: (sourceCode, filePath) => {
+        const spec = this.getPackageTranspilerSpecForFilePath(filePath)
+        if (spec) {
+          return this.transpileWithPackageTranspiler(sourceCode, filePath, spec)
+        }
+
+        return transpiler.compile(sourceCode, filePath)
+      },
+
+      shouldCompile: (sourceCode, filePath) => {
+        if (this.transpilerPaths[filePath]) {
+          return false
+        }
+        const spec = this.getPackageTranspilerSpecForFilePath(filePath)
+        if (spec) {
+          return true
+        }
+
+        return transpiler.shouldCompile(sourceCode, filePath)
+      }
+    }
+  }
+
+  getPackageTranspilerSpecForFilePath (filePath) {
+    if (this.specByFilePath[filePath] !== undefined) return this.specByFilePath[filePath]
+
+    // ignore node_modules
+    if (filePath.indexOf(path.sep + 'node_modules' + path.sep) > -1) {
+      return false
+    }
+
+    let thisPath = filePath
+    let lastPath = null
+    // Iterate parents from the file path to the root, checking at each level
+    // to see if a package manages transpilation for that directory.
+    // This means searching for a config for `/path/to/file/here.js` only
+    // only iterates four times, even if there are hundreds of configs registered.
+    while (thisPath !== lastPath) { // until we reach the root
+      let config = this.configByPackagePath[thisPath]
+      if (config) {
+        for (let i = 0; i < config.specs.length; i++) {
+          const spec = config.specs[i]
+          if (minimatch(filePath, path.join(config.path, spec.glob))) {
+            spec._config = config
+            this.specByFilePath[filePath] = spec
+            return spec
+          }
+        }
+      }
+
+      lastPath = thisPath
+      thisPath = path.resolve(thisPath, '..')
+    }
+
+    this.specByFilePath[filePath] = null
+    return null
+  }
+
+  getCachePath (sourceCode, filePath, spec) {
+    const transpilerPath = this.getTranspilerPath(spec)
+    const transpilerSource = spec._transpilerSource || fs.readFileSync(transpilerPath, 'utf8')
+    spec._transpilerSource = transpilerSource
+    const transpiler = this.getTranspiler(spec)
+
+    let hash = crypto
+      .createHash('sha1')
+      .update(JSON.stringify(spec.options || {}))
+      .update(transpilerSource, 'utf8')
+      .update(sourceCode, 'utf8')
+
+    if (transpiler && transpiler.getCacheKeyData) {
+      const additionalCacheData = transpiler.getCacheKeyData(sourceCode, filePath, spec.options)
+      hash.update(additionalCacheData, 'utf8')
+    }
+
+    return path.join('package-transpile', spec._config.name, hash.digest('hex'))
+  }
+
+  transpileWithPackageTranspiler (sourceCode, filePath, spec) {
+    const transpiler = this.getTranspiler(spec)
+
+    if (transpiler) {
+      const result = transpiler.transpile(sourceCode, filePath, spec.options || {})
+      if (result === undefined || (result && result.code === undefined)) {
+        return sourceCode
+      } else if (result.code) {
+        return result.code.toString()
+      } else {
+        throw new Error('Could not find a property `.code` on the transpilation results of ' + filePath)
+      }
+    } else {
+      const err = new Error("Could not resolve transpiler '" + spec.transpiler + "' from '" + spec._config.path + "'")
+      throw err
+    }
+  }
+
+  getTranspilerPath (spec) {
+    Resolve = Resolve || require('resolve')
+    return Resolve.sync(spec.transpiler, {
+      basedir: spec._config.path,
+      extensions: Object.keys(require.extensions)
+    })
+  }
+
+  getTranspiler (spec) {
+    const transpilerPath = this.getTranspilerPath(spec)
+    if (transpilerPath) {
+      const transpiler = require(transpilerPath)
+      this.transpilerPaths[transpilerPath] = true
+      return transpiler
+    }
+  }
+}
+
+module.exports = PackageTranspilationRegistry

src/package.coffee

@@ -6,6 +6,7 @@ CSON = require 'season'
 fs = require 'fs-plus'
 {Emitter, CompositeDisposable} = require 'event-kit'
 
+CompileCache = require './compile-cache'
 ModuleCache = require './module-cache'
 ScopedProperties = require './scoped-properties'
 BufferedProcess = require './buffered-process'
@@ -86,6 +87,7 @@ class Package
         @loadStylesheets()
         @registerDeserializerMethods()
         @activateCoreStartupServices()
+        @registerTranspilerConfig()
         @configSchemaRegisteredOnLoad = @registerConfigSchemaFromMetadata()
         @settingsPromise = @loadSettings()
         if @shouldRequireMainModuleOnLoad() and not @mainModule?
@@ -94,6 +96,9 @@ class Package
         @handleError("Failed to load the #{@name} package", error)
     this
 
+  unload: ->
+    @unregisterTranspilerConfig()
+
   shouldRequireMainModuleOnLoad: ->
     not (
       @metadata.deserializers? or
@@ -247,6 +252,14 @@ class Package
           @activationDisposables.add @packageManager.serviceHub.consume(name, version, @mainModule[methodName].bind(@mainModule))
     return
 
+  registerTranspilerConfig: ->
+    if @metadata.atomTranspilers
+      CompileCache.addTranspilerConfigForPath(@path, @name, @metadata.atomTranspilers)
+
+  unregisterTranspilerConfig: ->
+    if @metadata.atomTranspilers
+      CompileCache.removeTranspilerConfigForPath(@path)
+
   loadKeymaps: ->
     if @bundledPackage and @packageManager.packagesCache[@name]?
       @keymaps = (["#{@packageManager.resourcePath}#{path.sep}#{keymapPath}", keymapObject] for keymapPath, keymapObject of @packageManager.packagesCache[@name].keymaps)