From 26e3cd7df26dfbfac2f752919023ac310fabe8cf Mon Sep 17 00:00:00 2001 From: Ben Newman Date: Fri, 4 Mar 2016 16:44:15 -0500 Subject: [PATCH] Make sure /npm/ contains all node_modules directories. --- tools/isobuild/bundler.js | 314 ++++++++++++++++++++--------- tools/isobuild/compiler-plugin.js | 12 +- tools/isobuild/compiler.js | 56 +++-- tools/isobuild/import-scanner.js | 28 ++- tools/isobuild/isopack.js | 72 ++++--- tools/isobuild/meteor-npm.js | 12 +- tools/isobuild/package-source.js | 24 ++- tools/isobuild/source-arch.js | 12 ++ tools/static-assets/server/boot.js | 42 +++- 9 files changed, 395 insertions(+), 177 deletions(-) diff --git a/tools/isobuild/bundler.js b/tools/isobuild/bundler.js index 71f5bf2e01..5fcaae8517 100644 --- a/tools/isobuild/bundler.js +++ b/tools/isobuild/bundler.js @@ -148,6 +148,7 @@ // somewhere (decoupling target type from architecture) but it can // wait until later. +var assert = require('assert'); var util = require('util'); var Fiber = require('fibers'); var _ = require('underscore'); @@ -156,6 +157,7 @@ var compiler = require('./compiler.js'); var PackageSource = require('./package-source.js'); import Builder from './builder.js'; var compilerPluginModule = require('./compiler-plugin.js'); +var meteorNpm = require('./meteor-npm.js'); var files = require('../fs/files.js'); var archinfo = require('../utils/archinfo.js'); @@ -175,11 +177,11 @@ exports.ignoreFiles = [ /^\.git\/$/ /* often has too many files to watch */ ]; -var rejectBadPath = function (p) { - if (p.match(/\.\./)) { +function rejectBadPath(p) { + if (p.startsWith("..")) { throw new Error("bad path: " + p); } -}; +} var stripLeadingSlash = function (p) { if (p.charAt(0) === '/') { @@ -210,23 +212,147 @@ exports._mainJsContents = [ // Represents a node_modules directory that we need to copy into the // bundle or otherwise make available at runtime. -var NodeModulesDirectory = function (options) { - var self = this; - // The absolute path (on local disk) to a directory that contains - // the built node_modules to use. - self.sourcePath = options.sourcePath; +export class NodeModulesDirectory { + constructor({ + packageName, + sourceRoot, + sourcePath, + preferredBundlePath, + local = false, + npmDiscards = null, + writePackageJSON = false, + }) { + // Name of the package this node_modules directory belongs to, or null + // if it belongs to an application. Sometimes undefined when we don't + // know what package it belongs to, e.g. when reading node_modules + // from a program.json file. + assert.ok(typeof packageName === "string" || + typeof packageName === "undefined" || + packageName === null); + this.packageName = packageName; - // The path (relative to the bundle root) where we would preferably - // like the node_modules to be output (essentially cosmetic). - self.preferredBundlePath = options.preferredBundlePath; + // The absolute path of the root directory of the app or package that + // contains this node_modules directory. + assert.strictEqual(typeof sourceRoot, "string"); + this.sourceRoot = sourceRoot; - // Optionally, files to discard. - self.npmDiscards = options.npmDiscards; + // The absolute path (on local disk) to a directory that contains + // the built node_modules to use. + assert.strictEqual(typeof sourcePath, "string"); + this.sourcePath = sourcePath; - // Write a package.json file instead of copying the full directory. - self.writePackageJSON = !!options.writePackageJSON; -}; + // The path (relative to the bundle root) where we would preferably + // like the node_modules to be output. + this.preferredBundlePath = preferredBundlePath; + + // Boolean indicating whether the node_modules directory is locally + // accessible from other modules in the app or package. + this.local = !! local; + + // Optionally, files to discard. + this.npmDiscards = npmDiscards; + + // Write a package.json file instead of copying the full directory. + this.writePackageJSON = writePackageJSON; + } + + copy() { + return new this.constructor(this); + } + + isPortable() { + return meteorNpm.dependenciesArePortable(this.sourcePath); + } + + getPreferredBundlePath(kind) { + assert.ok(kind === "bundle" || + kind === "isopack", + kind); + + // Though we allow this.packageName to be undefined in the + // constructor, we must know the package that contains this + // node_modules directory to call getPreferredBundlePath. + const name = this.packageName; + assert.ok(typeof name === "string" || name === null); + + let relPath = files.pathRelative(this.sourceRoot, this.sourcePath); + rejectBadPath(relPath); + + const isApp = ! name; + if (! isApp) { + const relParts = relPath.split(files.pathSep); + + if (relParts[0] === ".npm" && + /^p(ackage|lugin)$/.test(relParts[1])) { + // Normalize .npm/package/node_modules/... paths so that they get + // copied into the bundle as if they were in the top-level local + // node_modules directory of the package. + relPath = relParts.slice(2).join(files.pathSep); + } else if (relParts[0] === "npm") { + const rootParts = this.sourceRoot.split(files.pathSep); + if (rootParts.indexOf(".meteor") >= 0) { + // If this.sourceRoot is inside a .meteor directory, then the + // npm/ at the beginning of the relPath must have been added by + // a previous call to getPreferredBundlePath, so we remove it + // here to avoid duplication. + relParts.shift(); + } + } + + if (kind === "bundle") { + relPath = files.pathJoin( + "node_modules", + "meteor", + colonConverter.convert(name), + ...relParts + ); + } + } + + // It's important not to put node_modules at the top level, so that it + // will not be visible from within plugins. + return files.pathJoin("npm", relPath); + } + + toJSON() { + return { + packageName: this.packageName, + writePackageJSON: this.writePackageJSON, + local: this.local, + }; + } + + // Returns an object mapping from relative bundle paths to the kind of + // objects returned by the toJSON method above. Note that this works + // even if the node_modules parameter is a string, though that will only + // be the case for bundles built before Meteor 1.3. + static readDirsFromJSON(node_modules, callerInfo) { + assert.strictEqual(typeof callerInfo.sourceRoot, "string"); + + const nodeModulesDirectories = Object.create(null); + + function add(moreInfo, relPath) { + rejectBadPath(relPath); + const sourcePath = files.pathJoin(callerInfo.sourceRoot, relPath); + nodeModulesDirectories[sourcePath] = new NodeModulesDirectory({ + ...callerInfo, + ...moreInfo, + sourcePath, + }); + } + + if (typeof node_modules === "string") { + // Old-style node_modules strings were only ever for + // .npm/package/node_modules directories, which are non-local. + add({ local: false }, node_modules); + } else if (node_modules) { + _.each(node_modules, add); + } + + return nodeModulesDirectories; + } +} /////////////////////////////////////////////////////////////////////////////// // File @@ -276,11 +402,11 @@ class File { // cached forever? Only makes sense of self.url is set. this.cacheable = options.cacheable || false; - // The node_modules directory that Npm.require() should search when - // called from inside this file, given as a NodeModulesDirectory, or - // null if Npm.depend() is not in effect for this file. Only works - // in the "server" architecture. - this.nodeModulesDirectory = null; + // The node_modules directories that Npm.require() should search when + // called from inside this file. Only includes non-local node_modules + // directories (e.g. .npm/package/node_modules), and only works on the + // server architecture. + this.nodeModulesDirectories = Object.create(null); // For server JS only. Assets associated with this slice; map from the path // that is the argument to Assets.getBinary, to a Buffer that is its contents. @@ -467,7 +593,7 @@ class Target { // The NodeModulesDirectory objects in this map are de-duplicated // aliases to the objects in the nodeModulesDirectory fields of // the File objects in this.js. - this.nodeModulesDirectories = {}; + this.nodeModulesDirectories = Object.create(null); // Static assets to include in the bundle. List of File. // For client targets, these are served over HTTP. @@ -838,25 +964,7 @@ class Target { f.setAssets(unibuildAssets); } - if (! isApp && unibuild.nodeModulesPath) { - var nmd = this.nodeModulesDirectories[unibuild.nodeModulesPath]; - if (! nmd) { - nmd = new NodeModulesDirectory({ - sourcePath: unibuild.nodeModulesPath, - // It's important that this path end with - // node_modules. Otherwise, if two modules in this package - // depend on each other, they won't be able to find each - // other! - preferredBundlePath: files.pathJoin( - 'npm', - colonConverter.convert(unibuild.pkg.name), - 'node_modules'), - npmDiscards: unibuild.pkg.npmDiscards - }); - this.nodeModulesDirectories[unibuild.nodeModulesPath] = nmd; - } - f.nodeModulesDirectory = nmd; - + _.each(unibuild.nodeModulesDirectories, nmd => { if (!archinfo.matches(this.arch, unibuild.arch)) { // The unibuild we're trying to include doesn't work for the // bundle target (eg, os.osx.x86_64 instead of os.linux.x86_64)! @@ -881,9 +989,14 @@ class Target { unibuild.pkg.name + ": missing .npm-shrinkwrap.json"); return; } + + nmd = nmd.copy(); nmd.writePackageJSON = true; } - } + + this.nodeModulesDirectories[nmd.sourcePath] = nmd; + f.nodeModulesDirectories[nmd.sourcePath] = nmd; + }); } // Both CSS and JS files can have source maps @@ -1283,8 +1396,9 @@ class JsImage { // Array of objects with keys: // - targetPath: relative path to use if saved to disk (or for stack traces) // - source: JS source code to load, as a string - // - nodeModulesDirectory: a NodeModulesDirectory indicating which - // directory should be searched by Npm.require() + // - nodeModulesDirectories: map from absolute node_modules directory + // paths to NodeModulesDirectory objects indicating which + // directories should be searched by Npm.require() // - sourceMap: if set, source map for this code, as a string // note: this can't be called `load` at it would shadow `load()` this.jsToLoad = []; @@ -1297,7 +1411,7 @@ class JsImage { // The NodeModulesDirectory objects in this map are de-duplicated // aliases to the objects in the nodeModulesDirectory fields of // the objects in this.jsToLoad. - this.nodeModulesDirectories = {}; + this.nodeModulesDirectories = Object.create(null); // Architecture required by this image this.arch = null; @@ -1377,19 +1491,29 @@ class JsImage { Package: ret, Npm: { require: function (name) { - if (! item.nodeModulesDirectory) { - // No Npm.depends associated with this package - return require(name); - } + let fullPath; - var nodeModuleDir = - files.pathJoin(item.nodeModulesDirectory.sourcePath, name); - var nodeModuleTopDir = - files.pathJoin(item.nodeModulesDirectory.sourcePath, - name.split("/")[0]); + _.some(item.nodeModulesDirectories, nmd => { + if (nmd.local) { + // Npm.require doesn't consider local node_modules + // directories. + return false; + } - if (files.exists(nodeModuleTopDir)) { - return require(files.convertToOSPath(nodeModuleDir)); + var nodeModulesTopDir = files.pathJoin( + nmd.sourcePath, + name.split("/")[0] + ); + + if (files.exists(nodeModulesTopDir)) { + return fullPath = files.convertToOSPath( + files.pathJoin(nmd.sourcePath, name) + ); + } + }); + + if (fullPath) { + return require(fullPath); } try { @@ -1475,21 +1599,14 @@ class JsImage { // Finalize choice of paths for node_modules directories -- These // paths are no longer just "preferred"; they are the final paths // that we will use - var nodeModulesDirectories = []; + var nodeModulesDirectories = Object.create(null); _.each(self.nodeModulesDirectories || [], function (nmd) { - // We do a little manipulation to make sure that generateFilename only - // adds suffixes to parts of the path other than the final node_modules, - // which needs to stay node_modules. - var dirname = files.pathDirname(nmd.preferredBundlePath); - var base = files.pathBasename(nmd.preferredBundlePath); - var generatedDir = builder.generateFilename(dirname, {directory: true}); - // We need to find the actual file system location for the node modules // this JS Image uses, so that we can add it to nodeModulesDirectories var modulesPhysicalLocation; if (! options.includeNodeModules || options.includeNodeModules === 'symlink') { - modulesPhysicalLocation = files.pathJoin(generatedDir, base); + modulesPhysicalLocation = nmd.getPreferredBundlePath("bundle"); } else if (options.includeNodeModules === 'reference-directly') { modulesPhysicalLocation = nmd.sourcePath; } else { @@ -1500,12 +1617,9 @@ class JsImage { "or 'reference-directly'. It was: " + options.includeNodeModules); } - nodeModulesDirectories.push(new NodeModulesDirectory({ - sourcePath: nmd.sourcePath, - preferredBundlePath: modulesPhysicalLocation, - npmDiscards: nmd.npmDiscards, - writePackageJSON: nmd.writePackageJSON - })); + nmd = nmd.copy(); + nmd.preferredBundlePath = modulesPhysicalLocation; + nodeModulesDirectories[nmd.sourcePath] = nmd; }); // If multiple load files share the same asset, only write one copy of @@ -1519,21 +1633,26 @@ class JsImage { throw new Error("No targetPath?"); } - var loadItem = {}; + var loadItem = { + node_modules: {} + }; - if (item.nodeModulesDirectory) { + _.each(item.nodeModulesDirectories, nmd => { // We need to make sure to use the directory name we got from // builder.generateFilename here. // XXX these two parallel data structures of self.jsToLoad and // self.nodeModulesDirectories are confusing - var generatedNMD = _.findWhere( - nodeModulesDirectories, - {sourcePath: item.nodeModulesDirectory.sourcePath} - ); + const generatedNMD = nodeModulesDirectories[nmd.sourcePath]; if (generatedNMD) { - loadItem.node_modules = generatedNMD.preferredBundlePath; + assert.strictEqual( + typeof generatedNMD.preferredBundlePath, + "string" + ); + + loadItem.node_modules[generatedNMD.preferredBundlePath] = + generatedNMD.toJSON(); } - } + }); if (item.sourceMap) { // Reference the source map in the source. Looked up later by @@ -1598,6 +1717,8 @@ class JsImage { // arch-specific code then the target will end up having an // appropriately specific arch. _.each(nodeModulesDirectories, function (nmd) { + assert.strictEqual(typeof nmd.preferredBundlePath, "string"); + if (nmd.writePackageJSON) { // Make sure there's an empty node_modules directory at the right place // in the tree (so that npm install puts modules there instead of @@ -1669,26 +1790,28 @@ class JsImage { _.each(json.load, function (item) { rejectBadPath(item.path); - var nmd = undefined; + let nodeModulesDirectories; if (item.node_modules) { - rejectBadPath(item.node_modules); - var node_modules = files.pathJoin(dir, item.node_modules); - if (! (node_modules in ret.nodeModulesDirectories)) { - ret.nodeModulesDirectories[node_modules] = - new NodeModulesDirectory({ - sourcePath: node_modules, - preferredBundlePath: item.node_modules - // No npmDiscards, because we should have already discarded things - // when writing the image to disk. - }); - } - nmd = ret.nodeModulesDirectories[node_modules]; + _.extend( + ret.nodeModulesDirectories, + nodeModulesDirectories = + // If item.node_modules is just a string, rather than an + // object with information about more than one node_modules + // directory, then we have no good way of knowing the + // packageName, but that's fine because we don't need to copy + // this node_modules directory into the bundle, so we + // shouldn't need to call getPreferredBundlePath. + NodeModulesDirectory.readDirsFromJSON(item.node_modules, { + packageName: files.pathBasename(dir), + sourceRoot: dir + }) + ); } var loadItem = { targetPath: item.path, source: files.readFile(files.pathJoin(dir, item.path), 'utf8'), - nodeModulesDirectory: nmd + nodeModulesDirectories, }; if (item.sourceMap) { @@ -1740,7 +1863,7 @@ class JsImageTarget extends Target { ret.jsToLoad.push({ targetPath: file.targetPath, source: file.contents().toString('utf8'), - nodeModulesDirectory: file.nodeModulesDirectory, + nodeModulesDirectories: file.nodeModulesDirectories, assets: file.assets, sourceMap: file.sourceMap, sourceMapRoot: file.sourceMapRoot @@ -2498,7 +2621,8 @@ exports.buildJsImage = Profile("bundler.buildJsImage", function (options) { // url path and not a file system path serveRoot: options.serveRoot || '/', npmDependencies: options.npmDependencies, - npmDir: options.npmDir + npmDir: options.npmDir, + localNodeModulesDirs: options.localNodeModulesDirs, }); var isopack = compiler.compile(packageSource, { diff --git a/tools/isobuild/compiler-plugin.js b/tools/isobuild/compiler-plugin.js index f2855e275f..7f4d768fa7 100644 --- a/tools/isobuild/compiler-plugin.js +++ b/tools/isobuild/compiler-plugin.js @@ -696,13 +696,23 @@ export class PackageSourceBatch { return; } + const nodeModulesPaths = []; + _.each(batch.unibuild.nodeModulesDirectories, (nmd, sourcePath) => { + if (! nmd.local) { + // Local node_modules directories will be found by the + // ImportScanner, but we need to tell it about any external + // node_modules directories (e.g. .npm/package/node_modules). + nodeModulesPaths.push(sourcePath); + } + }); + const scanner = new ImportScanner({ name, bundleArch: batch.processor.arch, extensions: batch.importExtensions, sourceRoot: batch.sourceRoot, usedPackageNames: batch.usedPackageNames, - nodeModulesPath: batch.unibuild.nodeModulesPath, + nodeModulesPaths, watchSet: batch.unibuild.watchSet, }); diff --git a/tools/isobuild/compiler.js b/tools/isobuild/compiler.js index 0d2e73e425..c4a8ffd10d 100644 --- a/tools/isobuild/compiler.js +++ b/tools/isobuild/compiler.js @@ -114,18 +114,15 @@ compiler.compile = Profile(function (packageSource, options) { // // We run this even if we have no dependencies, because we might // need to delete dependencies we used to have. - var isPortable = true; var nodeModulesPath = null; if (packageSource.npmCacheDirectory) { if (meteorNpm.updateDependencies(packageSource.name, packageSource.npmCacheDirectory, packageSource.npmDependencies)) { - nodeModulesPath = files.pathJoin(packageSource.npmCacheDirectory, - 'node_modules'); - if (! process.env.METEOR_FORCE_PORTABLE && - ! meteorNpm.dependenciesArePortable(packageSource.npmCacheDirectory)) { - isPortable = false; - } + nodeModulesPath = files.pathJoin( + packageSource.npmCacheDirectory, + 'node_modules' + ); } } @@ -181,9 +178,9 @@ compiler.compile = Profile(function (packageSource, options) { sourceArch: architecture, isopackCache: isopackCache, nodeModulesPath: nodeModulesPath, - isPortable: isPortable, noLineNumbers: options.noLineNumbers }); + _.extend(pluginProviderPackageNames, unibuildResult.pluginProviderPackageNames); }); @@ -335,7 +332,6 @@ var compileUnibuild = Profile(function (options) { const inputSourceArch = options.sourceArch; const isopackCache = options.isopackCache; const nodeModulesPath = options.nodeModulesPath; - const isPortable = options.isPortable; const noLineNumbers = options.noLineNumbers; const isApp = ! inputSourceArch.pkg.name; @@ -399,7 +395,34 @@ var compileUnibuild = Profile(function (options) { const sources = sourceProcessorFiles.sources || []; const assets = sourceProcessorFiles.assets || []; + const nodeModulesDirectories = Object.create(null); + + function addNodeModulesDirectory(options) { + const nmd = new bundler.NodeModulesDirectory(options); + nodeModulesDirectories[nmd.sourcePath] = nmd; + } + + Object.keys(inputSourceArch.localNodeModulesDirs).forEach(dir => { + addNodeModulesDirectory({ + packageName: inputSourceArch.pkg.name, + sourceRoot: inputSourceArch.sourceRoot, + sourcePath: files.pathJoin(inputSourceArch.sourceRoot, dir), + // Npm.strip applies to local node_modules directories of Meteor + // packages, as well as .npm/package/node_modules directories. + npmDiscards: isopk.npmDiscards, + local: true, + }); + }); + if (nodeModulesPath) { + addNodeModulesDirectory({ + packageName: inputSourceArch.pkg.name, + sourceRoot: inputSourceArch.sourceRoot, + sourcePath: nodeModulesPath, + npmDiscards: isopk.npmDiscards, + local: false, + }); + // If this slice has node modules, we should consider the shrinkwrap file // to be part of its inputs. (This is a little racy because there's no // guarantee that what we read here is precisely the version that's used, @@ -438,7 +461,7 @@ var compileUnibuild = Profile(function (options) { // Add all assets _.values(assets).forEach((asset) => { const relPath = asset.relPath; - const absPath = files.pathResolve(inputSourceArch.pkg.sourceRoot, relPath); + const absPath = files.pathResolve(inputSourceArch.sourceRoot, relPath); // readAndWatchFileWithHash returns an object carrying a buffer with the // file-contents. The buffer contains the original data of the file (no EOL @@ -454,7 +477,7 @@ var compileUnibuild = Profile(function (options) { _.values(sources).forEach((source) => { const relPath = source.relPath; const fileOptions = _.clone(source.fileOptions) || {}; - const absPath = files.pathResolve(inputSourceArch.pkg.sourceRoot, relPath); + const absPath = files.pathResolve(inputSourceArch.sourceRoot, relPath); const filename = files.pathBasename(relPath); // Find the handler for source files with this extension @@ -590,6 +613,9 @@ api.addAssets('${relPath}', 'client').`); return _.pick(symbol, ['name', 'testOnly']); }); + const isPortable = process.env.METEOR_FORCE_PORTABLE || + _.every(nodeModulesDirectories, nmd => nmd.isPortable()); + // *** Consider npm dependencies and portability var arch = inputSourceArch.arch; if (arch === "os" && ! isPortable) { @@ -597,10 +623,10 @@ api.addAssets('${relPath}', 'client').`); arch = archinfo.host(); } - let nodeModulesPathOrUndefined = nodeModulesPath; + let nodeModulesDirsOrUndefined = nodeModulesDirectories; if (! archinfo.matches(arch, "os") && ! isPortable) { // non-portable npm modules only work on server architectures - nodeModulesPathOrUndefined = undefined; + nodeModulesDirsOrUndefined = undefined; } // *** Output unibuild object @@ -610,7 +636,7 @@ api.addAssets('${relPath}', 'client').`); uses: inputSourceArch.uses, implies: inputSourceArch.implies, watchSet: watchSet, - nodeModulesPath: nodeModulesPathOrUndefined, + nodeModulesDirectories: nodeModulesDirsOrUndefined, declaredExports: declaredExports, resources: resources }); @@ -711,7 +737,7 @@ function runLinters({inputSourceArch, isopackCache, sources, // Read the file and add it to the WatchSet. const {hash, contents} = watch.readAndWatchFileWithHash( watchSet, - files.pathResolve(inputSourceArch.pkg.sourceRoot, relPath)); + files.pathResolve(inputSourceArch.sourceRoot, relPath)); const wrappedSource = { relPath, contents, hash, arch: inputSourceArch.arch, diff --git a/tools/isobuild/import-scanner.js b/tools/isobuild/import-scanner.js index a067e0febb..894e1335db 100644 --- a/tools/isobuild/import-scanner.js +++ b/tools/isobuild/import-scanner.js @@ -78,7 +78,7 @@ export default class ImportScanner { extensions = [".js", ".json"], sourceRoot, usedPackageNames = {}, - nodeModulesPath, + nodeModulesPaths = [], watchSet, }) { assert.ok(isString(sourceRoot)); @@ -88,7 +88,7 @@ export default class ImportScanner { this.extensions = extensions; this.sourceRoot = sourceRoot; this.usedPackageNames = usedPackageNames; - this.nodeModulesPath = nodeModulesPath; + this.nodeModulesPaths = nodeModulesPaths; this.watchSet = watchSet; this.absPathToOutputIndex = Object.create(null); this.allMissingNodeModules = Object.create(null); @@ -372,12 +372,13 @@ export default class ImportScanner { } _getNodeModulesInstallPath(absPath) { - if (this.nodeModulesPath) { - const relPathWithinNodeModules = - pathRelative(this.nodeModulesPath, absPath); + let installPath; + + this.nodeModulesPaths.some(path => { + const relPathWithinNodeModules = pathRelative(path, absPath); if (relPathWithinNodeModules.startsWith("..")) { - // absPath is not a subdirectory of this.nodeModulesPath. + // absPath is not a subdirectory of path. return; } @@ -389,8 +390,13 @@ export default class ImportScanner { // Install the module into the local node_modules directory within // this app or package. - return pathJoin("node_modules", relPathWithinNodeModules); - } + return installPath = pathJoin( + "node_modules", + relPathWithinNodeModules + ); + }); + + return installPath; } _getSourceRootInstallPath(absPath) { @@ -571,10 +577,12 @@ export default class ImportScanner { dir = pathDirname(dir); } - if (! resolved && this.nodeModulesPath) { + if (! resolved) { // After checking any local node_modules directories, fall back to // the package NPM directory, if one was specified. - resolved = this._joinAndStat(this.nodeModulesPath, id); + this.nodeModulesPaths.some(path => { + return resolved = this._joinAndStat(path, id); + }); } if (! resolved) { diff --git a/tools/isobuild/isopack.js b/tools/isobuild/isopack.js index 42e911bb62..b8348becaf 100644 --- a/tools/isobuild/isopack.js +++ b/tools/isobuild/isopack.js @@ -1,3 +1,4 @@ +var assert = require('assert'); var compiler = require('./compiler.js'); var archinfo = require('../utils/archinfo.js'); var _ = require('underscore'); @@ -30,7 +31,7 @@ var rejectBadPath = function (p) { // - uses // - implies // - watchSet -// - nodeModulesPath +// - nodeModulesDirectories // - declaredExports // - resources @@ -93,10 +94,9 @@ var Unibuild = function (isopack, options) { // against at build time. null if matched against a specific filename. self.resources = options.resources; - // Absolute path to the node_modules directory to use at runtime to - // resolve Npm.require() calls in this unibuild. null if this unibuild - // does not have a node_modules. - self.nodeModulesPath = options.nodeModulesPath; + // Map from absolute paths of node_modules directories to + // NodeModulesDirectory objects. + self.nodeModulesDirectories = options.nodeModulesDirectories; }; /////////////////////////////////////////////////////////////////////////////// @@ -982,13 +982,6 @@ _.extend(Isopack.prototype, { var unibuildHasPrelink = unibuildJson.format === "unipackage-unibuild-pre1"; - var nodeModulesPath = null; - if (unibuildJson.node_modules) { - rejectBadPath(unibuildJson.node_modules); - nodeModulesPath = - files.pathJoin(unibuildBasePath, unibuildJson.node_modules); - } - var resources = []; _.each(unibuildJson.resources, function (resource) { @@ -1080,6 +1073,12 @@ _.extend(Isopack.prototype, { } }); + const nodeModulesDirectories = + bundler.NodeModulesDirectory.readDirsFromJSON( + unibuildJson.node_modules, + { packageName: self.name, + sourceRoot: unibuildBasePath }); + self.unibuilds.push(new Unibuild(self, { // At some point we stopped writing 'kind's to the metadata file, so // default to main. @@ -1088,7 +1087,7 @@ _.extend(Isopack.prototype, { uses: unibuildJson.uses, implies: unibuildJson.implies, watchSet: unibuildWatchSets[unibuildMeta.path], - nodeModulesPath: nodeModulesPath, + nodeModulesDirectories, declaredExports: declaredExports, resources: resources })); @@ -1225,7 +1224,7 @@ _.extend(Isopack.prototype, { // have been separated?), but also there can be different sets of // directories as well (eg, for a isopack merged with from multiple // isopacks with _loadUnibuildsFromPath). - var npmDirectories = {}; + const npmDirsToCopy = Object.create(null); // Pre-linker versions of Meteor expect all packages in the warehouse to // contain a file called "package.js"; they use this as part of deciding @@ -1269,20 +1268,17 @@ _.extend(Isopack.prototype, { } // Figure out where the npm dependencies go. - var nodeModulesPath = undefined; - var needToCopyNodeModules = false; - if (unibuild.nodeModulesPath) { - if (_.has(npmDirectories, unibuild.nodeModulesPath)) { + const node_modules = {}; + _.each(unibuild.nodeModulesDirectories, nmd => { + const bundlePath = _.has(npmDirsToCopy, nmd.sourcePath) // We already have this npm directory from another unibuild. - nodeModulesPath = npmDirectories[unibuild.nodeModulesPath]; - } else { - // It's important not to put node_modules at the top level of the - // isopack, so that it is not visible from within plugins. - nodeModulesPath = npmDirectories[unibuild.nodeModulesPath] = - builder.generateFilename("npm/node_modules", {directory: true}); - needToCopyNodeModules = true; - } - } + ? npmDirsToCopy[nmd.sourcePath] + : npmDirsToCopy[nmd.sourcePath] = builder.generateFilename( + nmd.getPreferredBundlePath("isopack"), + { directory: true } + ); + node_modules[bundlePath] = nmd.toJSON(); + }); // Construct unibuild metadata var unibuildJson = { @@ -1299,7 +1295,7 @@ _.extend(Isopack.prototype, { }; }), implies: (_.isEmpty(unibuild.implies) ? undefined : unibuild.implies), - node_modules: nodeModulesPath, + node_modules, resources: [] }; @@ -1377,16 +1373,6 @@ _.extend(Isopack.prototype, { }); }); - // If unibuild has included node_modules, copy them in - if (needToCopyNodeModules) { - builder.copyDirectory({ - from: unibuild.nodeModulesPath, - to: nodeModulesPath, - npmDiscards: self.npmDiscards, - symlink: false - }); - } - // Control file for unibuild builder.writeJson(unibuildJsonFile, unibuildJson); unibuildInfos.push({ @@ -1396,6 +1382,16 @@ _.extend(Isopack.prototype, { }); }); + // If unibuilds included node_modules, copy them in. + _.each(npmDirsToCopy, (bundlePath, sourcePath) => { + builder.copyDirectory({ + from: sourcePath, + to: bundlePath, + npmDiscards: self.npmDiscards, + symlink: false + }); + }); + // Plugins _.each(self.plugins, function (pluginsByArch, name) { _.each(pluginsByArch, function (plugin) { diff --git a/tools/isobuild/meteor-npm.js b/tools/isobuild/meteor-npm.js index dcec99f17b..b7e618c7ba 100644 --- a/tools/isobuild/meteor-npm.js +++ b/tools/isobuild/meteor-npm.js @@ -3,6 +3,7 @@ /// and a variety of related commands. Notably, we use `npm shrinkwrap` /// to ensure we get consistent versions of npm sub-dependencies. +var assert = require('assert'); var cleanup = require('../tool-env/cleanup.js'); var files = require('../fs/files.js'); var os = require('os'); @@ -117,14 +118,19 @@ meteorNpm.updateDependencies = function (packageName, // expect it to work, rather than containing native extensions that // were built just for our architecture), else // false. updateDependencies should first be used to bring -// packageNpmDir up to date. -meteorNpm.dependenciesArePortable = function (packageNpmDir) { +// nodeModulesDir up to date. +meteorNpm.dependenciesArePortable = function (nodeModulesDir) { // We use a simple heuristic: we check to see if a package (or any // of its transitive depedencies) contains any *.node files. .node // is the extension that signals to Node that it should load a file // as a shared object rather than as JavaScript, so this should work // in the vast majority of cases. + assert.strictEqual( + files.pathBasename(nodeModulesDir), + "node_modules" + ); + var search = function (dir) { return _.find(files.readdir(dir), function (itemName) { if (itemName.match(/\.node$/)) { @@ -137,7 +143,7 @@ meteorNpm.dependenciesArePortable = function (packageNpmDir) { }) || false; }; - return ! search(files.pathJoin(packageNpmDir, 'node_modules')); + return ! search(nodeModulesDir); }; var makeNewPackageNpmDir = function (newPackageNpmDir) { diff --git a/tools/isobuild/package-source.js b/tools/isobuild/package-source.js index 840279c9d5..abf7a7eeba 100644 --- a/tools/isobuild/package-source.js +++ b/tools/isobuild/package-source.js @@ -367,6 +367,7 @@ _.extend(PackageSource.prototype, { // - npmDependencies // - cordovaDependencies // - npmDir + // - localNodeModulesDirs initFromOptions: function (name, options) { var self = this; self.name = name; @@ -403,6 +404,7 @@ _.extend(PackageSource.prototype, { const sourceArch = new SourceArch(self, { kind: options.kind, arch: "os", + sourceRoot: self.sourceRoot, uses: _.map(options.use, splitConstraint), getFiles() { return { @@ -411,6 +413,11 @@ _.extend(PackageSource.prototype, { } }); + if (options.localNodeModulesDirs) { + _.extend(sourceArch.localNodeModulesDirs, + options.localNodeModulesDirs); + } + self.architectures.push(sourceArch); if (! self._checkCrossUnibuildVersionConstraints()) { @@ -1190,6 +1197,7 @@ _.extend(PackageSource.prototype, { self.architectures.push(new SourceArch(self, { kind: "main", arch: arch, + sourceRoot: self.sourceRoot, uses: api.uses[arch], implies: api.implies[arch], getFiles(sourceProcessorSet, watchSet) { @@ -1206,7 +1214,7 @@ _.extend(PackageSource.prototype, { self._findSources({ sourceProcessorSet, watchSet, - arch, + sourceArch: this, isApp: false }).forEach(relPath => { if (! _.has(relPathToSourceObj, relPath)) { @@ -1282,6 +1290,7 @@ _.extend(PackageSource.prototype, { var sourceArch = new SourceArch(self, { kind: 'app', arch: arch, + sourceRoot: self.sourceRoot, uses: uses, getFiles(sourceProcessorSet, watchSet) { sourceProcessorSet.watchSet = watchSet; @@ -1289,7 +1298,7 @@ _.extend(PackageSource.prototype, { const findOptions = { sourceProcessorSet, watchSet, - arch, + sourceArch: this, ignoreFiles, isApp: true, loopChecker: new SymlinkLoopChecker(self.sourceRoot) @@ -1376,10 +1385,11 @@ _.extend(PackageSource.prototype, { sourceProcessorSet, watchSet, isApp, - arch, + sourceArch, loopChecker = new SymlinkLoopChecker(this.sourceRoot), ignoreFiles = [] }) { + const arch = sourceArch.arch; const sourceReadOptions = sourceProcessorSet.appReadDirectoryOptions(arch); @@ -1414,7 +1424,6 @@ _.extend(PackageSource.prototype, { ); const anyLevelExcludes = [ - /^node_modules\/$/, /^tests\/$/, arch === "os" ? /^client\/$/ : /^server\/$/, ...sourceReadOptions.exclude, @@ -1440,6 +1449,10 @@ _.extend(PackageSource.prototype, { while (!_.isEmpty(sourceDirectories)) { var dir = sourceDirectories.shift(); + if (/(^|\/)node_modules\/$/.test(dir)) { + sourceArch.localNodeModulesDirs[dir] = true; + continue; + } // remove trailing slash dir = dir.substr(0, dir.length - 1); @@ -1469,11 +1482,12 @@ _.extend(PackageSource.prototype, { sourceProcessorSet, watchSet, isApp, - arch, + sourceArch, loopChecker = new SymlinkLoopChecker(this.sourceRoot), ignoreFiles = [], }) { // Now look for assets for this unibuild. + const arch = sourceArch.arch; const assetDir = archinfo.matches(arch, "web") ? "public/" : "private/"; var assetDirs = this._readAndWatchDirectory('', watchSet, { names: [assetDir] diff --git a/tools/isobuild/source-arch.js b/tools/isobuild/source-arch.js index 2d461deeaf..342e27e7a6 100644 --- a/tools/isobuild/source-arch.js +++ b/tools/isobuild/source-arch.js @@ -14,6 +14,7 @@ export default class SourceArch { constructor(pkg, { kind, // required arch, // required + sourceRoot, // required getFiles, // required uses = [], implies = [], @@ -24,6 +25,7 @@ export default class SourceArch { }) { isString(kind) || reportMissingOption('kind'); isString(arch) || reportMissingOption('arch'); + isString(sourceRoot) || reportMissingOption('sourceRoot'); isFunction(getFiles) || reportMissingOption('getFiles'); this.pkg = pkg; @@ -36,6 +38,9 @@ export default class SourceArch { // unibuild. this.arch = arch; + // Absolute path of the root directory of this package or application. + this.sourceRoot = sourceRoot; + // Packages used. The ordering is significant only for determining // import symbol priority (it doesn't affect load order), and a given // package could appear more than once in the list, so code that @@ -88,6 +93,13 @@ export default class SourceArch { // this package) to compute this. this.getFiles = getFiles; + // Object whose keys are relative paths of local node_modules + // directories in this package or application, for the given + // architecture. Does not include the .npm/package/node_modules + // directory installed by Npm.depends. Should be populated when + // getFiles is called. + this.localNodeModulesDirs = Object.create(null); + // Symbols that this architecture should export. List of symbols (as // strings). this.declaredExports = declaredExports; diff --git a/tools/static-assets/server/boot.js b/tools/static-assets/server/boot.js index b39ac44519..246039a959 100644 --- a/tools/static-assets/server/boot.js +++ b/tools/static-assets/server/boot.js @@ -139,6 +139,23 @@ var startCheckForLiveParent = function (parentPid) { Fiber(function () { _.each(serverJson.load, function (fileInfo) { var code = fs.readFileSync(path.resolve(serverDir, fileInfo.path)); + var nonLocalNodeModulesPaths = []; + + function addNodeModulesPath(path) { + nonLocalNodeModulesPaths.push( + files.pathResolve(serverDir, path) + ); + } + + if (typeof fileInfo.node_modules === "string") { + addNodeModulesPath(fileInfo.node_modules); + } else if (fileInfo.node_modules) { + _.each(fileInfo.node_modules, function (info, path) { + if (! info.local) { + addNodeModulesPath(path); + } + }); + } var Npm = { /** @@ -149,20 +166,25 @@ Fiber(function () { * @memberOf Npm */ require: function (name) { - if (! fileInfo.node_modules) { + if (nonLocalNodeModulesPaths.length === 0) { return require(name); } - var nodeModuleBase = path.resolve(serverDir, - files.convertToOSPath(fileInfo.node_modules)); - var nodeModuleDir = path.resolve(nodeModuleBase, name); + var fullPath; - // If the user does `Npm.require('foo/bar')`, then we should resolve to - // the package's node modules if `foo` was one of the modules we - // installed. (`foo/bar` might be implemented as `foo/bar.js` so we - // can't just naively see if all of nodeModuleDir exists. - if (fs.existsSync(path.resolve(nodeModuleBase, name.split("/")[0]))) { - return require(nodeModuleDir); + nonLocalNodeModulesPaths.some(function (nodeModuleBase) { + var packageBase = files.pathResolve( + nodeModuleBase, + name.split("/", 1)[0] + ); + + if (fs.existsSync(packageBase)) { + return fullPath = files.pathResolve(nodeModuleBase, name); + } + }); + + if (fullPath) { + return require(fullPath); } try {