diff --git a/package.json b/package.json index 2fef7fbc1..b26a914b4 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "less-cache": "0.23", "line-top-index": "0.2.0", "marked": "^0.3.6", + "minimatch": "^3.0.3", "mocha": "2.5.1", "normalize-package-data": "^2.0.0", "nslog": "^3", diff --git a/spec/package-transpilation-registry-spec.js b/spec/package-transpilation-registry-spec.js new file mode 100644 index 000000000..cec52a949 --- /dev/null +++ b/spec/package-transpilation-registry-spec.js @@ -0,0 +1,145 @@ +/** @babel */ +import fs from 'fs' +import path from 'path' + +import {it, fit, ffit, fffit, beforeEach, afterEach} from './async-spec-helpers' + +import PackageTranspilationRegistry from '../src/package-transpilation-registry' + +const originalCompiler = { + getCachePath: (sourceCode, filePath) => { + return "orig-cache-path" + }, + + compile: (sourceCode, filePath) => { + return sourceCode + "-original-compiler" + }, + + shouldCompile: (sourceCode, filePath) => { + return path.extname(filePath) === '.js' + } +} + +describe("PackageTranspilationRegistry", () => { + let registry + let wrappedCompiler + + beforeEach(() => { + registry = new PackageTranspilationRegistry() + wrappedCompiler = registry.wrapTranspiler(originalCompiler) + }) + + it('falls through to the original compiler by default', () => { + spyOn(originalCompiler, 'getCachePath') + spyOn(originalCompiler, 'compile') + spyOn(originalCompiler, 'shouldCompile') + + wrappedCompiler.getCachePath('source', '/path/to/file.js') + wrappedCompiler.compile('source', '/path/to/filejs') + wrappedCompiler.shouldCompile('source', '/path/to/file.js') + + expect(originalCompiler.getCachePath).toHaveBeenCalled() + expect(originalCompiler.compile).toHaveBeenCalled() + expect(originalCompiler.shouldCompile).toHaveBeenCalled() + }) + + describe('when a file is contained in a path that has custom transpilation', () => { + const hitPath = '/path/to/lib/file.js' + const hitPathCoffee = '/path/to/file2.coffee' + const missPath = '/path/other/file3.js' + const hitPathMissSubdir = '/path/to/file4.js' + const hitPathMissExt = '/path/to/file5.ts' + const nodeModulesFolder = '/path/to/lib/node_modules/file6.js' + const hitNonStandardExt = '/path/to/file7.omgwhatisthis' + + const jsSpec = { glob: "lib/**/*.js", transpiler: './transpiler-js', options: { type: 'js' } } + const coffeeSpec = { glob: "*.coffee", transpiler: './transpiler-coffee', options: { type: 'coffee' } } + const omgSpec = { glob: "*.omgwhatisthis", transpiler: './transpiler-omg', options: { type: 'omg' } } + + const jsTranspiler = { + transpile: (sourceCode, filePath, options) => { + return {code: sourceCode + "-transpiler-js"} + }, + + getCacheKeyData: (sourceCode, filePath, options) => { + return 'js-transpiler-cache-data' + } + } + + const coffeeTranspiler = { + transpile: (sourceCode, filePath, options) => { + return {code: sourceCode + "-transpiler-coffee"} + }, + + getCacheKeyData: (sourceCode, filePath, options) => { + return 'coffee-transpiler-cache-data' + } + } + + const omgTranspiler = { + transpile: (sourceCode, filePath, options) => { + return {code: sourceCode + "-transpiler-omg"} + }, + + getCacheKeyData: (sourceCode, filePath, options) => { + return 'omg-transpiler-cache-data' + } + } + + beforeEach(() => { + jsSpec._transpilerSource = "js-transpiler-source" + coffeeSpec._transpilerSource = "coffee-transpiler-source" + omgTranspiler._transpilerSource = "omg-transpiler-source" + + spyOn(registry, "getTranspiler").andCallFake(spec => { + if (spec.transpiler === './transpiler-js') return jsTranspiler + if (spec.transpiler === './transpiler-coffee') return coffeeTranspiler + if (spec.transpiler === './transpiler-omg') return omgTranspiler + throw new Error('bad transpiler path ' + spec.transpiler) + }) + + registry.addTranspilerConfigForPath('/path/to', 'my-package', [ + jsSpec, coffeeSpec, omgSpec + ]) + }) + + it('always returns true from shouldCompile for a file in that dir that match a glob', () => { + spyOn(originalCompiler, 'shouldCompile').andReturn(false) + expect(wrappedCompiler.shouldCompile('source', hitPath)).toBe(true) + expect(wrappedCompiler.shouldCompile('source', hitPathCoffee)).toBe(true) + expect(wrappedCompiler.shouldCompile('source', hitNonStandardExt)).toBe(true) + expect(wrappedCompiler.shouldCompile('source', hitPathMissExt)).toBe(false) + expect(wrappedCompiler.shouldCompile('source', hitPathMissSubdir)).toBe(false) + expect(wrappedCompiler.shouldCompile('source', missPath)).toBe(false) + expect(wrappedCompiler.shouldCompile('source', nodeModulesFolder)).toBe(false) + }) + + it('calls getCacheKeyData on the transpiler to get additional cache key data', () => { + spyOn(registry, "getTranspilerPath").andReturn("./transpiler-js") + spyOn(jsTranspiler, 'getCacheKeyData').andCallThrough() + + wrappedCompiler.getCachePath('source', missPath, jsSpec) + expect(jsTranspiler.getCacheKeyData).not.toHaveBeenCalled() + wrappedCompiler.getCachePath('source', hitPath, jsSpec) + expect(jsTranspiler.getCacheKeyData).toHaveBeenCalled() + }) + + it('compiles files matching a glob with the associated transpiler, and the old one otherwise', () => { + spyOn(jsTranspiler, "transpile").andCallThrough() + spyOn(coffeeTranspiler, "transpile").andCallThrough() + spyOn(omgTranspiler, "transpile").andCallThrough() + + expect(wrappedCompiler.compile('source', hitPath)).toEqual('source-transpiler-js') + expect(jsTranspiler.transpile).toHaveBeenCalledWith('source', hitPath, jsSpec.options) + expect(wrappedCompiler.compile('source', hitPathCoffee)).toEqual('source-transpiler-coffee') + expect(coffeeTranspiler.transpile).toHaveBeenCalledWith('source', hitPathCoffee, coffeeSpec.options) + expect(wrappedCompiler.compile('source', hitNonStandardExt)).toEqual('source-transpiler-omg') + expect(omgTranspiler.transpile).toHaveBeenCalledWith('source', hitNonStandardExt, omgSpec.options) + + expect(wrappedCompiler.compile('source', missPath)).toEqual('source-original-compiler') + expect(wrappedCompiler.compile('source', hitPathMissExt)).toEqual('source-original-compiler') + expect(wrappedCompiler.compile('source', hitPathMissSubdir)).toEqual('source-original-compiler') + expect(wrappedCompiler.compile('source', nodeModulesFolder)).toEqual('source-original-compiler') + }) + }) +}) diff --git a/src/compile-cache.js b/src/compile-cache.js index ad1bd0a85..fed890d5c 100644 --- a/src/compile-cache.js +++ b/src/compile-cache.js @@ -7,12 +7,26 @@ var path = require('path') var fs = require('fs-plus') + +var PackageTranspilationRegistry = require('./package-transpilation-registry') var CSON = null +var packageTranspilationRegistry = new PackageTranspilationRegistry() + var COMPILERS = { - '.js': require('./babel'), - '.ts': require('./typescript'), - '.coffee': require('./coffee-script') + '.js': packageTranspilationRegistry.wrapTranspiler(require('./babel')), + '.ts': packageTranspilationRegistry.wrapTranspiler(require('./typescript')), + '.coffee': packageTranspilationRegistry.wrapTranspiler(require('./coffee-script')) +} + +exports.addTranspilerConfigForPath = function (packagePath, packageName, config) { + packagePath = fs.realpathSync(packagePath) + packageTranspilationRegistry.addTranspilerConfigForPath(packagePath, packageName, config) +} + +exports.removeTranspilerConfigForPath = function (packagePath) { + packagePath = fs.realpathSync(packagePath) + packageTranspilationRegistry.removeTranspilerConfigForPath(packagePath) } var cacheStats = {} diff --git a/src/package-transpilation-registry.js b/src/package-transpilation-registry.js new file mode 100644 index 000000000..46509807f --- /dev/null +++ b/src/package-transpilation-registry.js @@ -0,0 +1,161 @@ +'use strict' +// This file is required by compile-cache, which is required directly from +// apm, so it can only use the subset of newer JavaScript features that apm's +// version of Node supports. Strict mode is required for block scoped declarations. + +const crypto = require('crypto') +const fs = require('fs') +const path = require('path') + +const minimatch = require('minimatch') + +let Resolve = null + +class PackageTranspilationRegistry { + constructor () { + this.configByPackagePath = {} + this.specByFilePath = {} + this.transpilerPaths = {} + } + + addTranspilerConfigForPath (packagePath, packageName, config) { + this.configByPackagePath[packagePath] = { + name: packageName, + path: packagePath, + specs: config + } + } + + removeTranspilerConfigForPath (packagePath) { + delete this.configByPackagePath[packagePath] + } + + // Wraps the transpiler in an object with the same interface + // that falls back to the original transpiler implementation if and + // only if a package hasn't registered its desire to transpile its own source. + wrapTranspiler (transpiler) { + return { + getCachePath: (sourceCode, filePath) => { + const spec = this.getPackageTranspilerSpecForFilePath(filePath) + if (spec) { + return this.getCachePath(sourceCode, filePath, spec) + } + + return transpiler.getCachePath(sourceCode, filePath) + }, + + compile: (sourceCode, filePath) => { + const spec = this.getPackageTranspilerSpecForFilePath(filePath) + if (spec) { + return this.transpileWithPackageTranspiler(sourceCode, filePath, spec) + } + + return transpiler.compile(sourceCode, filePath) + }, + + shouldCompile: (sourceCode, filePath) => { + if (this.transpilerPaths[filePath]) { + return false + } + const spec = this.getPackageTranspilerSpecForFilePath(filePath) + if (spec) { + return true + } + + return transpiler.shouldCompile(sourceCode, filePath) + } + } + } + + getPackageTranspilerSpecForFilePath (filePath) { + if (this.specByFilePath[filePath] !== undefined) return this.specByFilePath[filePath] + + // ignore node_modules + if (filePath.indexOf(path.sep + 'node_modules' + path.sep) > -1) { + return false + } + + let thisPath = filePath + let lastPath = null + // Iterate parents from the file path to the root, checking at each level + // to see if a package manages transpilation for that directory. + // This means searching for a config for `/path/to/file/here.js` only + // only iterates four times, even if there are hundreds of configs registered. + while (thisPath !== lastPath) { // until we reach the root + let config = this.configByPackagePath[thisPath] + if (config) { + for (let i = 0; i < config.specs.length; i++) { + const spec = config.specs[i] + if (minimatch(filePath, path.join(config.path, spec.glob))) { + spec._config = config + this.specByFilePath[filePath] = spec + return spec + } + } + } + + lastPath = thisPath + thisPath = path.resolve(thisPath, '..') + } + + this.specByFilePath[filePath] = null + return null + } + + getCachePath (sourceCode, filePath, spec) { + const transpilerPath = this.getTranspilerPath(spec) + const transpilerSource = spec._transpilerSource || fs.readFileSync(transpilerPath, 'utf8') + spec._transpilerSource = transpilerSource + const transpiler = this.getTranspiler(spec) + + let hash = crypto + .createHash('sha1') + .update(JSON.stringify(spec.options || {})) + .update(transpilerSource, 'utf8') + .update(sourceCode, 'utf8') + + if (transpiler && transpiler.getCacheKeyData) { + const additionalCacheData = transpiler.getCacheKeyData(sourceCode, filePath, spec.options) + hash.update(additionalCacheData, 'utf8') + } + + return path.join('package-transpile', spec._config.name, hash.digest('hex')) + } + + transpileWithPackageTranspiler (sourceCode, filePath, spec) { + const transpiler = this.getTranspiler(spec) + + if (transpiler) { + const result = transpiler.transpile(sourceCode, filePath, spec.options || {}) + if (result === undefined || (result && result.code === undefined)) { + return sourceCode + } else if (result.code) { + return result.code.toString() + } else { + throw new Error('Could not find a property `.code` on the transpilation results of ' + filePath) + } + } else { + const err = new Error("Could not resolve transpiler '" + spec.transpiler + "' from '" + spec._config.path + "'") + throw err + } + } + + getTranspilerPath (spec) { + Resolve = Resolve || require('resolve') + return Resolve.sync(spec.transpiler, { + basedir: spec._config.path, + extensions: Object.keys(require.extensions) + }) + } + + getTranspiler (spec) { + const transpilerPath = this.getTranspilerPath(spec) + if (transpilerPath) { + const transpiler = require(transpilerPath) + this.transpilerPaths[transpilerPath] = true + return transpiler + } + } +} + +module.exports = PackageTranspilationRegistry diff --git a/src/package.coffee b/src/package.coffee index 323dfa8d2..6029c99ae 100644 --- a/src/package.coffee +++ b/src/package.coffee @@ -6,6 +6,7 @@ CSON = require 'season' fs = require 'fs-plus' {Emitter, CompositeDisposable} = require 'event-kit' +CompileCache = require './compile-cache' ModuleCache = require './module-cache' ScopedProperties = require './scoped-properties' BufferedProcess = require './buffered-process' @@ -86,6 +87,7 @@ class Package @loadStylesheets() @registerDeserializerMethods() @activateCoreStartupServices() + @registerTranspilerConfig() @configSchemaRegisteredOnLoad = @registerConfigSchemaFromMetadata() @settingsPromise = @loadSettings() if @shouldRequireMainModuleOnLoad() and not @mainModule? @@ -94,6 +96,9 @@ class Package @handleError("Failed to load the #{@name} package", error) this + unload: -> + @unregisterTranspilerConfig() + shouldRequireMainModuleOnLoad: -> not ( @metadata.deserializers? or @@ -247,6 +252,14 @@ class Package @activationDisposables.add @packageManager.serviceHub.consume(name, version, @mainModule[methodName].bind(@mainModule)) return + registerTranspilerConfig: -> + if @metadata.atomTranspilers + CompileCache.addTranspilerConfigForPath(@path, @name, @metadata.atomTranspilers) + + unregisterTranspilerConfig: -> + if @metadata.atomTranspilers + CompileCache.removeTranspilerConfigForPath(@path) + loadKeymaps: -> if @bundledPackage and @packageManager.packagesCache[@name]? @keymaps = (["#{@packageManager.resourcePath}#{path.sep}#{keymapPath}", keymapObject] for keymapPath, keymapObject of @packageManager.packagesCache[@name].keymaps)