github/meteor
1
1
Fork 0
mirror of https://github.com/meteor/meteor.git synced 2026-05-02 03:01:46 -04:00
Code Issues Packages Projects Releases Wiki Activity

Make sure <bundle>/npm/ contains all node_modules directories.

Browse Source
This commit is contained in:
Ben Newman
2016-03-04 16:44:15 -05:00
parent 99213da719
commit 26e3cd7df2
9 changed files with 395 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files

314
tools/isobuild/bundler.js
View File

@@ -148,6 +148,7 @@
// somewhere (decoupling target type from architecture) but it can
// wait until later.
var assert = require('assert');
var util = require('util');
var Fiber = require('fibers');
var _ = require('underscore');
@@ -156,6 +157,7 @@ var compiler = require('./compiler.js');
var PackageSource = require('./package-source.js');
import Builder from './builder.js';
var compilerPluginModule = require('./compiler-plugin.js');
var meteorNpm = require('./meteor-npm.js');
var files = require('../fs/files.js');
var archinfo = require('../utils/archinfo.js');
@@ -175,11 +177,11 @@ exports.ignoreFiles = [
/^\.git\/$/ /* often has too many files to watch */
];
var rejectBadPath = function (p) {
if (p.match(/\.\./)) {
function rejectBadPath(p) {
if (p.startsWith("..")) {
throw new Error("bad path: " + p);
}
};
}
var stripLeadingSlash = function (p) {
if (p.charAt(0) === '/') {
@@ -210,23 +212,147 @@ exports._mainJsContents = [
// Represents a node_modules directory that we need to copy into the
// bundle or otherwise make available at runtime.
var NodeModulesDirectory = function (options) {
var self = this;
// The absolute path (on local disk) to a directory that contains
// the built node_modules to use.
self.sourcePath = options.sourcePath;
export class NodeModulesDirectory {
constructor({
packageName,
sourceRoot,
sourcePath,
preferredBundlePath,
local = false,
npmDiscards = null,
writePackageJSON = false,
}) {
// Name of the package this node_modules directory belongs to, or null
// if it belongs to an application. Sometimes undefined when we don't
// know what package it belongs to, e.g. when reading node_modules
// from a program.json file.
assert.ok(typeof packageName === "string" ||
typeof packageName === "undefined" ||
packageName === null);
this.packageName = packageName;
// The path (relative to the bundle root) where we would preferably
// like the node_modules to be output (essentially cosmetic).
self.preferredBundlePath = options.preferredBundlePath;
// The absolute path of the root directory of the app or package that
// contains this node_modules directory.
assert.strictEqual(typeof sourceRoot, "string");
this.sourceRoot = sourceRoot;
// Optionally, files to discard.
self.npmDiscards = options.npmDiscards;
// The absolute path (on local disk) to a directory that contains
// the built node_modules to use.
assert.strictEqual(typeof sourcePath, "string");
this.sourcePath = sourcePath;
// Write a package.json file instead of copying the full directory.
self.writePackageJSON = !!options.writePackageJSON;
};
// The path (relative to the bundle root) where we would preferably
// like the node_modules to be output.
this.preferredBundlePath = preferredBundlePath;
// Boolean indicating whether the node_modules directory is locally
// accessible from other modules in the app or package.
this.local = !! local;
// Optionally, files to discard.
this.npmDiscards = npmDiscards;
// Write a package.json file instead of copying the full directory.
this.writePackageJSON = writePackageJSON;
}
copy() {
return new this.constructor(this);
}
isPortable() {
return meteorNpm.dependenciesArePortable(this.sourcePath);
}
getPreferredBundlePath(kind) {
assert.ok(kind === "bundle" ||
kind === "isopack",
kind);
// Though we allow this.packageName to be undefined in the
// constructor, we must know the package that contains this
// node_modules directory to call getPreferredBundlePath.
const name = this.packageName;
assert.ok(typeof name === "string" || name === null);
let relPath = files.pathRelative(this.sourceRoot, this.sourcePath);
rejectBadPath(relPath);
const isApp = ! name;
if (! isApp) {
const relParts = relPath.split(files.pathSep);
if (relParts[0] === ".npm" &&
/^p(ackage|lugin)$/.test(relParts[1])) {
// Normalize .npm/package/node_modules/... paths so that they get
// copied into the bundle as if they were in the top-level local
// node_modules directory of the package.
relPath = relParts.slice(2).join(files.pathSep);
} else if (relParts[0] === "npm") {
const rootParts = this.sourceRoot.split(files.pathSep);
if (rootParts.indexOf(".meteor") >= 0) {
// If this.sourceRoot is inside a .meteor directory, then the
// npm/ at the beginning of the relPath must have been added by
// a previous call to getPreferredBundlePath, so we remove it
// here to avoid duplication.
relParts.shift();
}
}
if (kind === "bundle") {
relPath = files.pathJoin(
"node_modules",
"meteor",
colonConverter.convert(name),
...relParts
);
}
}
// It's important not to put node_modules at the top level, so that it
// will not be visible from within plugins.
return files.pathJoin("npm", relPath);
}
toJSON() {
return {
packageName: this.packageName,
writePackageJSON: this.writePackageJSON,
local: this.local,
};
}
// Returns an object mapping from relative bundle paths to the kind of
// objects returned by the toJSON method above. Note that this works
// even if the node_modules parameter is a string, though that will only
// be the case for bundles built before Meteor 1.3.
static readDirsFromJSON(node_modules, callerInfo) {
assert.strictEqual(typeof callerInfo.sourceRoot, "string");
const nodeModulesDirectories = Object.create(null);
function add(moreInfo, relPath) {
rejectBadPath(relPath);
const sourcePath = files.pathJoin(callerInfo.sourceRoot, relPath);
nodeModulesDirectories[sourcePath] = new NodeModulesDirectory({
...callerInfo,
...moreInfo,
sourcePath,
});
}
if (typeof node_modules === "string") {
// Old-style node_modules strings were only ever for
// .npm/package/node_modules directories, which are non-local.
add({ local: false }, node_modules);
} else if (node_modules) {
_.each(node_modules, add);
}
return nodeModulesDirectories;
}
}
///////////////////////////////////////////////////////////////////////////////
// File
@@ -276,11 +402,11 @@ class File {
// cached forever? Only makes sense of self.url is set.
this.cacheable = options.cacheable || false;
// The node_modules directory that Npm.require() should search when
// called from inside this file, given as a NodeModulesDirectory, or
// null if Npm.depend() is not in effect for this file. Only works
// in the "server" architecture.
this.nodeModulesDirectory = null;
// The node_modules directories that Npm.require() should search when
// called from inside this file. Only includes non-local node_modules
// directories (e.g. .npm/package/node_modules), and only works on the
// server architecture.
this.nodeModulesDirectories = Object.create(null);
// For server JS only. Assets associated with this slice; map from the path
// that is the argument to Assets.getBinary, to a Buffer that is its contents.
@@ -467,7 +593,7 @@ class Target {
// The NodeModulesDirectory objects in this map are de-duplicated
// aliases to the objects in the nodeModulesDirectory fields of
// the File objects in this.js.
this.nodeModulesDirectories = {};
this.nodeModulesDirectories = Object.create(null);
// Static assets to include in the bundle. List of File.
// For client targets, these are served over HTTP.
@@ -838,25 +964,7 @@ class Target {
f.setAssets(unibuildAssets);
}
if (! isApp && unibuild.nodeModulesPath) {
var nmd = this.nodeModulesDirectories[unibuild.nodeModulesPath];
if (! nmd) {
nmd = new NodeModulesDirectory({
sourcePath: unibuild.nodeModulesPath,
// It's important that this path end with
// node_modules. Otherwise, if two modules in this package
// depend on each other, they won't be able to find each
// other!
preferredBundlePath: files.pathJoin(
'npm',
colonConverter.convert(unibuild.pkg.name),
'node_modules'),
npmDiscards: unibuild.pkg.npmDiscards
});
this.nodeModulesDirectories[unibuild.nodeModulesPath] = nmd;
}
f.nodeModulesDirectory = nmd;
_.each(unibuild.nodeModulesDirectories, nmd => {
if (!archinfo.matches(this.arch, unibuild.arch)) {
// The unibuild we're trying to include doesn't work for the
// bundle target (eg, os.osx.x86_64 instead of os.linux.x86_64)!
@@ -881,9 +989,14 @@ class Target {
unibuild.pkg.name + ": missing .npm-shrinkwrap.json");
return;
}
nmd = nmd.copy();
nmd.writePackageJSON = true;
}
}
this.nodeModulesDirectories[nmd.sourcePath] = nmd;
f.nodeModulesDirectories[nmd.sourcePath] = nmd;
});
}
// Both CSS and JS files can have source maps
@@ -1283,8 +1396,9 @@ class JsImage {
// Array of objects with keys:
// - targetPath: relative path to use if saved to disk (or for stack traces)
// - source: JS source code to load, as a string
// - nodeModulesDirectory: a NodeModulesDirectory indicating which
// directory should be searched by Npm.require()
// - nodeModulesDirectories: map from absolute node_modules directory
// paths to NodeModulesDirectory objects indicating which
// directories should be searched by Npm.require()
// - sourceMap: if set, source map for this code, as a string
// note: this can't be called `load` at it would shadow `load()`
this.jsToLoad = [];
@@ -1297,7 +1411,7 @@ class JsImage {
// The NodeModulesDirectory objects in this map are de-duplicated
// aliases to the objects in the nodeModulesDirectory fields of
// the objects in this.jsToLoad.
this.nodeModulesDirectories = {};
this.nodeModulesDirectories = Object.create(null);
// Architecture required by this image
this.arch = null;
@@ -1377,19 +1491,29 @@ class JsImage {
Package: ret,
Npm: {
require: function (name) {
if (! item.nodeModulesDirectory) {
// No Npm.depends associated with this package
return require(name);
}
let fullPath;
var nodeModuleDir =
files.pathJoin(item.nodeModulesDirectory.sourcePath, name);
var nodeModuleTopDir =
files.pathJoin(item.nodeModulesDirectory.sourcePath,
name.split("/")[0]);
_.some(item.nodeModulesDirectories, nmd => {
if (nmd.local) {
// Npm.require doesn't consider local node_modules
// directories.
return false;
}
if (files.exists(nodeModuleTopDir)) {
return require(files.convertToOSPath(nodeModuleDir));
var nodeModulesTopDir = files.pathJoin(
nmd.sourcePath,
name.split("/")[0]
);
if (files.exists(nodeModulesTopDir)) {
return fullPath = files.convertToOSPath(
files.pathJoin(nmd.sourcePath, name)
);
}
});
if (fullPath) {
return require(fullPath);
}
try {
@@ -1475,21 +1599,14 @@ class JsImage {
// Finalize choice of paths for node_modules directories -- These
// paths are no longer just "preferred"; they are the final paths
// that we will use
var nodeModulesDirectories = [];
var nodeModulesDirectories = Object.create(null);
_.each(self.nodeModulesDirectories || [], function (nmd) {
// We do a little manipulation to make sure that generateFilename only
// adds suffixes to parts of the path other than the final node_modules,
// which needs to stay node_modules.
var dirname = files.pathDirname(nmd.preferredBundlePath);
var base = files.pathBasename(nmd.preferredBundlePath);
var generatedDir = builder.generateFilename(dirname, {directory: true});
// We need to find the actual file system location for the node modules
// this JS Image uses, so that we can add it to nodeModulesDirectories
var modulesPhysicalLocation;
if (! options.includeNodeModules ||
options.includeNodeModules === 'symlink') {
modulesPhysicalLocation = files.pathJoin(generatedDir, base);
modulesPhysicalLocation = nmd.getPreferredBundlePath("bundle");
} else if (options.includeNodeModules === 'reference-directly') {
modulesPhysicalLocation = nmd.sourcePath;
} else {
@@ -1500,12 +1617,9 @@ class JsImage {
"or 'reference-directly'. It was: " + options.includeNodeModules);
}
nodeModulesDirectories.push(new NodeModulesDirectory({
sourcePath: nmd.sourcePath,
preferredBundlePath: modulesPhysicalLocation,
npmDiscards: nmd.npmDiscards,
writePackageJSON: nmd.writePackageJSON
}));
nmd = nmd.copy();
nmd.preferredBundlePath = modulesPhysicalLocation;
nodeModulesDirectories[nmd.sourcePath] = nmd;
});
// If multiple load files share the same asset, only write one copy of
@@ -1519,21 +1633,26 @@ class JsImage {
throw new Error("No targetPath?");
}
var loadItem = {};
var loadItem = {
node_modules: {}
};
if (item.nodeModulesDirectory) {
_.each(item.nodeModulesDirectories, nmd => {
// We need to make sure to use the directory name we got from
// builder.generateFilename here.
// XXX these two parallel data structures of self.jsToLoad and
// self.nodeModulesDirectories are confusing
var generatedNMD = _.findWhere(
nodeModulesDirectories,
{sourcePath: item.nodeModulesDirectory.sourcePath}
);
const generatedNMD = nodeModulesDirectories[nmd.sourcePath];
if (generatedNMD) {
loadItem.node_modules = generatedNMD.preferredBundlePath;
assert.strictEqual(
typeof generatedNMD.preferredBundlePath,
"string"
);
loadItem.node_modules[generatedNMD.preferredBundlePath] =
generatedNMD.toJSON();
}
}
});
if (item.sourceMap) {
// Reference the source map in the source. Looked up later by
@@ -1598,6 +1717,8 @@ class JsImage {
// arch-specific code then the target will end up having an
// appropriately specific arch.
_.each(nodeModulesDirectories, function (nmd) {
assert.strictEqual(typeof nmd.preferredBundlePath, "string");
if (nmd.writePackageJSON) {
// Make sure there's an empty node_modules directory at the right place
// in the tree (so that npm install puts modules there instead of
@@ -1669,26 +1790,28 @@ class JsImage {
_.each(json.load, function (item) {
rejectBadPath(item.path);
var nmd = undefined;
let nodeModulesDirectories;
if (item.node_modules) {
rejectBadPath(item.node_modules);
var node_modules = files.pathJoin(dir, item.node_modules);
if (! (node_modules in ret.nodeModulesDirectories)) {
ret.nodeModulesDirectories[node_modules] =
new NodeModulesDirectory({
sourcePath: node_modules,
preferredBundlePath: item.node_modules
// No npmDiscards, because we should have already discarded things
// when writing the image to disk.
});
}
nmd = ret.nodeModulesDirectories[node_modules];
_.extend(
ret.nodeModulesDirectories,
nodeModulesDirectories =
// If item.node_modules is just a string, rather than an
// object with information about more than one node_modules
// directory, then we have no good way of knowing the
// packageName, but that's fine because we don't need to copy
// this node_modules directory into the bundle, so we
// shouldn't need to call getPreferredBundlePath.
NodeModulesDirectory.readDirsFromJSON(item.node_modules, {
packageName: files.pathBasename(dir),
sourceRoot: dir
})
);
}
var loadItem = {
targetPath: item.path,
source: files.readFile(files.pathJoin(dir, item.path), 'utf8'),
nodeModulesDirectory: nmd
nodeModulesDirectories,
};
if (item.sourceMap) {
@@ -1740,7 +1863,7 @@ class JsImageTarget extends Target {
ret.jsToLoad.push({
targetPath: file.targetPath,
source: file.contents().toString('utf8'),
nodeModulesDirectory: file.nodeModulesDirectory,
nodeModulesDirectories: file.nodeModulesDirectories,
assets: file.assets,
sourceMap: file.sourceMap,
sourceMapRoot: file.sourceMapRoot
@@ -2498,7 +2621,8 @@ exports.buildJsImage = Profile("bundler.buildJsImage", function (options) {
// url path and not a file system path
serveRoot: options.serveRoot || '/',
npmDependencies: options.npmDependencies,
npmDir: options.npmDir
npmDir: options.npmDir,
localNodeModulesDirs: options.localNodeModulesDirs,
});
var isopack = compiler.compile(packageSource, {

12
tools/isobuild/compiler-plugin.js
View File

@@ -696,13 +696,23 @@ export class PackageSourceBatch {
return;
}
const nodeModulesPaths = [];
_.each(batch.unibuild.nodeModulesDirectories, (nmd, sourcePath) => {
if (! nmd.local) {
// Local node_modules directories will be found by the
// ImportScanner, but we need to tell it about any external
// node_modules directories (e.g. .npm/package/node_modules).
nodeModulesPaths.push(sourcePath);
}
});
const scanner = new ImportScanner({
name,
bundleArch: batch.processor.arch,
extensions: batch.importExtensions,
sourceRoot: batch.sourceRoot,
usedPackageNames: batch.usedPackageNames,
nodeModulesPath: batch.unibuild.nodeModulesPath,
nodeModulesPaths,
watchSet: batch.unibuild.watchSet,
});

56
tools/isobuild/compiler.js
View File

@@ -114,18 +114,15 @@ compiler.compile = Profile(function (packageSource, options) {
//
// We run this even if we have no dependencies, because we might
// need to delete dependencies we used to have.
var isPortable = true;
var nodeModulesPath = null;
if (packageSource.npmCacheDirectory) {
if (meteorNpm.updateDependencies(packageSource.name,
packageSource.npmCacheDirectory,
packageSource.npmDependencies)) {
nodeModulesPath = files.pathJoin(packageSource.npmCacheDirectory,
'node_modules');
if (! process.env.METEOR_FORCE_PORTABLE &&
! meteorNpm.dependenciesArePortable(packageSource.npmCacheDirectory)) {
isPortable = false;
}
nodeModulesPath = files.pathJoin(
packageSource.npmCacheDirectory,
'node_modules'
);
}
}
@@ -181,9 +178,9 @@ compiler.compile = Profile(function (packageSource, options) {
sourceArch: architecture,
isopackCache: isopackCache,
nodeModulesPath: nodeModulesPath,
isPortable: isPortable,
noLineNumbers: options.noLineNumbers
});
_.extend(pluginProviderPackageNames,
unibuildResult.pluginProviderPackageNames);
});
@@ -335,7 +332,6 @@ var compileUnibuild = Profile(function (options) {
const inputSourceArch = options.sourceArch;
const isopackCache = options.isopackCache;
const nodeModulesPath = options.nodeModulesPath;
const isPortable = options.isPortable;
const noLineNumbers = options.noLineNumbers;
const isApp = ! inputSourceArch.pkg.name;
@@ -399,7 +395,34 @@ var compileUnibuild = Profile(function (options) {
const sources = sourceProcessorFiles.sources || [];
const assets = sourceProcessorFiles.assets || [];
const nodeModulesDirectories = Object.create(null);
function addNodeModulesDirectory(options) {
const nmd = new bundler.NodeModulesDirectory(options);
nodeModulesDirectories[nmd.sourcePath] = nmd;
}
Object.keys(inputSourceArch.localNodeModulesDirs).forEach(dir => {
addNodeModulesDirectory({
packageName: inputSourceArch.pkg.name,
sourceRoot: inputSourceArch.sourceRoot,
sourcePath: files.pathJoin(inputSourceArch.sourceRoot, dir),
// Npm.strip applies to local node_modules directories of Meteor
// packages, as well as .npm/package/node_modules directories.
npmDiscards: isopk.npmDiscards,
local: true,
});
});
if (nodeModulesPath) {
addNodeModulesDirectory({
packageName: inputSourceArch.pkg.name,
sourceRoot: inputSourceArch.sourceRoot,
sourcePath: nodeModulesPath,
npmDiscards: isopk.npmDiscards,
local: false,
});
// If this slice has node modules, we should consider the shrinkwrap file
// to be part of its inputs. (This is a little racy because there's no
// guarantee that what we read here is precisely the version that's used,
@@ -438,7 +461,7 @@ var compileUnibuild = Profile(function (options) {
// Add all assets
_.values(assets).forEach((asset) => {
const relPath = asset.relPath;
const absPath = files.pathResolve(inputSourceArch.pkg.sourceRoot, relPath);
const absPath = files.pathResolve(inputSourceArch.sourceRoot, relPath);
// readAndWatchFileWithHash returns an object carrying a buffer with the
// file-contents. The buffer contains the original data of the file (no EOL
@@ -454,7 +477,7 @@ var compileUnibuild = Profile(function (options) {
_.values(sources).forEach((source) => {
const relPath = source.relPath;
const fileOptions = _.clone(source.fileOptions) || {};
const absPath = files.pathResolve(inputSourceArch.pkg.sourceRoot, relPath);
const absPath = files.pathResolve(inputSourceArch.sourceRoot, relPath);
const filename = files.pathBasename(relPath);
// Find the handler for source files with this extension
@@ -590,6 +613,9 @@ api.addAssets('${relPath}', 'client').`);
return _.pick(symbol, ['name', 'testOnly']);
});
const isPortable = process.env.METEOR_FORCE_PORTABLE ||
_.every(nodeModulesDirectories, nmd => nmd.isPortable());
// *** Consider npm dependencies and portability
var arch = inputSourceArch.arch;
if (arch === "os" && ! isPortable) {
@@ -597,10 +623,10 @@ api.addAssets('${relPath}', 'client').`);
arch = archinfo.host();
}
let nodeModulesPathOrUndefined = nodeModulesPath;
let nodeModulesDirsOrUndefined = nodeModulesDirectories;
if (! archinfo.matches(arch, "os") && ! isPortable) {
// non-portable npm modules only work on server architectures
nodeModulesPathOrUndefined = undefined;
nodeModulesDirsOrUndefined = undefined;
}
// *** Output unibuild object
@@ -610,7 +636,7 @@ api.addAssets('${relPath}', 'client').`);
uses: inputSourceArch.uses,
implies: inputSourceArch.implies,
watchSet: watchSet,
nodeModulesPath: nodeModulesPathOrUndefined,
nodeModulesDirectories: nodeModulesDirsOrUndefined,
declaredExports: declaredExports,
resources: resources
});
@@ -711,7 +737,7 @@ function runLinters({inputSourceArch, isopackCache, sources,
// Read the file and add it to the WatchSet.
const {hash, contents} = watch.readAndWatchFileWithHash(
watchSet,
files.pathResolve(inputSourceArch.pkg.sourceRoot, relPath));
files.pathResolve(inputSourceArch.sourceRoot, relPath));
const wrappedSource = {
relPath, contents, hash,
arch: inputSourceArch.arch,

28
tools/isobuild/import-scanner.js
View File

@@ -78,7 +78,7 @@ export default class ImportScanner {
extensions = [".js", ".json"],
sourceRoot,
usedPackageNames = {},
nodeModulesPath,
nodeModulesPaths = [],
watchSet,
}) {
assert.ok(isString(sourceRoot));
@@ -88,7 +88,7 @@ export default class ImportScanner {
this.extensions = extensions;
this.sourceRoot = sourceRoot;
this.usedPackageNames = usedPackageNames;
this.nodeModulesPath = nodeModulesPath;
this.nodeModulesPaths = nodeModulesPaths;
this.watchSet = watchSet;
this.absPathToOutputIndex = Object.create(null);
this.allMissingNodeModules = Object.create(null);
@@ -372,12 +372,13 @@ export default class ImportScanner {
}
_getNodeModulesInstallPath(absPath) {
if (this.nodeModulesPath) {
const relPathWithinNodeModules =
pathRelative(this.nodeModulesPath, absPath);
let installPath;
this.nodeModulesPaths.some(path => {
const relPathWithinNodeModules = pathRelative(path, absPath);
if (relPathWithinNodeModules.startsWith("..")) {
// absPath is not a subdirectory of this.nodeModulesPath.
// absPath is not a subdirectory of path.
return;
}
@@ -389,8 +390,13 @@ export default class ImportScanner {
// Install the module into the local node_modules directory within
// this app or package.
return pathJoin("node_modules", relPathWithinNodeModules);
}
return installPath = pathJoin(
"node_modules",
relPathWithinNodeModules
);
});
return installPath;
}
_getSourceRootInstallPath(absPath) {
@@ -571,10 +577,12 @@ export default class ImportScanner {
dir = pathDirname(dir);
}
if (! resolved && this.nodeModulesPath) {
if (! resolved) {
// After checking any local node_modules directories, fall back to
// the package NPM directory, if one was specified.
resolved = this._joinAndStat(this.nodeModulesPath, id);
this.nodeModulesPaths.some(path => {
return resolved = this._joinAndStat(path, id);
});
}
if (! resolved) {

72
tools/isobuild/isopack.js
View File

@@ -1,3 +1,4 @@
var assert = require('assert');
var compiler = require('./compiler.js');
var archinfo = require('../utils/archinfo.js');
var _ = require('underscore');
@@ -30,7 +31,7 @@ var rejectBadPath = function (p) {
// - uses
// - implies
// - watchSet
// - nodeModulesPath
// - nodeModulesDirectories
// - declaredExports
// - resources
@@ -93,10 +94,9 @@ var Unibuild = function (isopack, options) {
// against at build time. null if matched against a specific filename.
self.resources = options.resources;
// Absolute path to the node_modules directory to use at runtime to
// resolve Npm.require() calls in this unibuild. null if this unibuild
// does not have a node_modules.
self.nodeModulesPath = options.nodeModulesPath;
// Map from absolute paths of node_modules directories to
// NodeModulesDirectory objects.
self.nodeModulesDirectories = options.nodeModulesDirectories;
};
///////////////////////////////////////////////////////////////////////////////
@@ -982,13 +982,6 @@ _.extend(Isopack.prototype, {
var unibuildHasPrelink =
unibuildJson.format === "unipackage-unibuild-pre1";
var nodeModulesPath = null;
if (unibuildJson.node_modules) {
rejectBadPath(unibuildJson.node_modules);
nodeModulesPath =
files.pathJoin(unibuildBasePath, unibuildJson.node_modules);
}
var resources = [];
_.each(unibuildJson.resources, function (resource) {
@@ -1080,6 +1073,12 @@ _.extend(Isopack.prototype, {
}
});
const nodeModulesDirectories =
bundler.NodeModulesDirectory.readDirsFromJSON(
unibuildJson.node_modules,
{ packageName: self.name,
sourceRoot: unibuildBasePath });
self.unibuilds.push(new Unibuild(self, {
// At some point we stopped writing 'kind's to the metadata file, so
// default to main.
@@ -1088,7 +1087,7 @@ _.extend(Isopack.prototype, {
uses: unibuildJson.uses,
implies: unibuildJson.implies,
watchSet: unibuildWatchSets[unibuildMeta.path],
nodeModulesPath: nodeModulesPath,
nodeModulesDirectories,
declaredExports: declaredExports,
resources: resources
}));
@@ -1225,7 +1224,7 @@ _.extend(Isopack.prototype, {
// have been separated?), but also there can be different sets of
// directories as well (eg, for a isopack merged with from multiple
// isopacks with _loadUnibuildsFromPath).
var npmDirectories = {};
const npmDirsToCopy = Object.create(null);
// Pre-linker versions of Meteor expect all packages in the warehouse to
// contain a file called "package.js"; they use this as part of deciding
@@ -1269,20 +1268,17 @@ _.extend(Isopack.prototype, {
}
// Figure out where the npm dependencies go.
var nodeModulesPath = undefined;
var needToCopyNodeModules = false;
if (unibuild.nodeModulesPath) {
if (_.has(npmDirectories, unibuild.nodeModulesPath)) {
const node_modules = {};
_.each(unibuild.nodeModulesDirectories, nmd => {
const bundlePath = _.has(npmDirsToCopy, nmd.sourcePath)
// We already have this npm directory from another unibuild.
nodeModulesPath = npmDirectories[unibuild.nodeModulesPath];
} else {
// It's important not to put node_modules at the top level of the
// isopack, so that it is not visible from within plugins.
nodeModulesPath = npmDirectories[unibuild.nodeModulesPath] =
builder.generateFilename("npm/node_modules", {directory: true});
needToCopyNodeModules = true;
}
}
? npmDirsToCopy[nmd.sourcePath]
: npmDirsToCopy[nmd.sourcePath] = builder.generateFilename(
nmd.getPreferredBundlePath("isopack"),
{ directory: true }
);
node_modules[bundlePath] = nmd.toJSON();
});
// Construct unibuild metadata
var unibuildJson = {
@@ -1299,7 +1295,7 @@ _.extend(Isopack.prototype, {
};
}),
implies: (_.isEmpty(unibuild.implies) ? undefined : unibuild.implies),
node_modules: nodeModulesPath,
node_modules,
resources: []
};
@@ -1377,16 +1373,6 @@ _.extend(Isopack.prototype, {
});
});
// If unibuild has included node_modules, copy them in
if (needToCopyNodeModules) {
builder.copyDirectory({
from: unibuild.nodeModulesPath,
to: nodeModulesPath,
npmDiscards: self.npmDiscards,
symlink: false
});
}
// Control file for unibuild
builder.writeJson(unibuildJsonFile, unibuildJson);
unibuildInfos.push({
@@ -1396,6 +1382,16 @@ _.extend(Isopack.prototype, {
});
});
// If unibuilds included node_modules, copy them in.
_.each(npmDirsToCopy, (bundlePath, sourcePath) => {
builder.copyDirectory({
from: sourcePath,
to: bundlePath,
npmDiscards: self.npmDiscards,
symlink: false
});
});
// Plugins
_.each(self.plugins, function (pluginsByArch, name) {
_.each(pluginsByArch, function (plugin) {

12
tools/isobuild/meteor-npm.js
View File

@@ -3,6 +3,7 @@
/// and a variety of related commands. Notably, we use `npm shrinkwrap`
/// to ensure we get consistent versions of npm sub-dependencies.
var assert = require('assert');
var cleanup = require('../tool-env/cleanup.js');
var files = require('../fs/files.js');
var os = require('os');
@@ -117,14 +118,19 @@ meteorNpm.updateDependencies = function (packageName,
// expect it to work, rather than containing native extensions that
// were built just for our architecture), else
// false. updateDependencies should first be used to bring
// packageNpmDir up to date.
meteorNpm.dependenciesArePortable = function (packageNpmDir) {
// nodeModulesDir up to date.
meteorNpm.dependenciesArePortable = function (nodeModulesDir) {
// We use a simple heuristic: we check to see if a package (or any
// of its transitive depedencies) contains any *.node files. .node
// is the extension that signals to Node that it should load a file
// as a shared object rather than as JavaScript, so this should work
// in the vast majority of cases.
assert.strictEqual(
files.pathBasename(nodeModulesDir),
"node_modules"
);
var search = function (dir) {
return _.find(files.readdir(dir), function (itemName) {
if (itemName.match(/\.node$/)) {
@@ -137,7 +143,7 @@ meteorNpm.dependenciesArePortable = function (packageNpmDir) {
}) || false;
};
return ! search(files.pathJoin(packageNpmDir, 'node_modules'));
return ! search(nodeModulesDir);
};
var makeNewPackageNpmDir = function (newPackageNpmDir) {

24
tools/isobuild/package-source.js
View File

@@ -367,6 +367,7 @@ _.extend(PackageSource.prototype, {
// - npmDependencies
// - cordovaDependencies
// - npmDir
// - localNodeModulesDirs
initFromOptions: function (name, options) {
var self = this;
self.name = name;
@@ -403,6 +404,7 @@ _.extend(PackageSource.prototype, {
const sourceArch = new SourceArch(self, {
kind: options.kind,
arch: "os",
sourceRoot: self.sourceRoot,
uses: _.map(options.use, splitConstraint),
getFiles() {
return {
@@ -411,6 +413,11 @@ _.extend(PackageSource.prototype, {
}
});
if (options.localNodeModulesDirs) {
_.extend(sourceArch.localNodeModulesDirs,
options.localNodeModulesDirs);
}
self.architectures.push(sourceArch);
if (! self._checkCrossUnibuildVersionConstraints()) {
@@ -1190,6 +1197,7 @@ _.extend(PackageSource.prototype, {
self.architectures.push(new SourceArch(self, {
kind: "main",
arch: arch,
sourceRoot: self.sourceRoot,
uses: api.uses[arch],
implies: api.implies[arch],
getFiles(sourceProcessorSet, watchSet) {
@@ -1206,7 +1214,7 @@ _.extend(PackageSource.prototype, {
self._findSources({
sourceProcessorSet,
watchSet,
arch,
sourceArch: this,
isApp: false
}).forEach(relPath => {
if (! _.has(relPathToSourceObj, relPath)) {
@@ -1282,6 +1290,7 @@ _.extend(PackageSource.prototype, {
var sourceArch = new SourceArch(self, {
kind: 'app',
arch: arch,
sourceRoot: self.sourceRoot,
uses: uses,
getFiles(sourceProcessorSet, watchSet) {
sourceProcessorSet.watchSet = watchSet;
@@ -1289,7 +1298,7 @@ _.extend(PackageSource.prototype, {
const findOptions = {
sourceProcessorSet,
watchSet,
arch,
sourceArch: this,
ignoreFiles,
isApp: true,
loopChecker: new SymlinkLoopChecker(self.sourceRoot)
@@ -1376,10 +1385,11 @@ _.extend(PackageSource.prototype, {
sourceProcessorSet,
watchSet,
isApp,
arch,
sourceArch,
loopChecker = new SymlinkLoopChecker(this.sourceRoot),
ignoreFiles = []
}) {
const arch = sourceArch.arch;
const sourceReadOptions =
sourceProcessorSet.appReadDirectoryOptions(arch);
@@ -1414,7 +1424,6 @@ _.extend(PackageSource.prototype, {
);
const anyLevelExcludes = [
/^node_modules\/$/,
/^tests\/$/,
arch === "os" ? /^client\/$/ : /^server\/$/,
...sourceReadOptions.exclude,
@@ -1440,6 +1449,10 @@ _.extend(PackageSource.prototype, {
while (!_.isEmpty(sourceDirectories)) {
var dir = sourceDirectories.shift();
if (/(^|\/)node_modules\/$/.test(dir)) {
sourceArch.localNodeModulesDirs[dir] = true;
continue;
}
// remove trailing slash
dir = dir.substr(0, dir.length - 1);
@@ -1469,11 +1482,12 @@ _.extend(PackageSource.prototype, {
sourceProcessorSet,
watchSet,
isApp,
arch,
sourceArch,
loopChecker = new SymlinkLoopChecker(this.sourceRoot),
ignoreFiles = [],
}) {
// Now look for assets for this unibuild.
const arch = sourceArch.arch;
const assetDir = archinfo.matches(arch, "web") ? "public/" : "private/";
var assetDirs = this._readAndWatchDirectory('', watchSet, {
names: [assetDir]

12
tools/isobuild/source-arch.js
View File

@@ -14,6 +14,7 @@ export default class SourceArch {
constructor(pkg, {
kind, // required
arch, // required
sourceRoot, // required
getFiles, // required
uses = [],
implies = [],
@@ -24,6 +25,7 @@ export default class SourceArch {
}) {
isString(kind) || reportMissingOption('kind');
isString(arch) || reportMissingOption('arch');
isString(sourceRoot) || reportMissingOption('sourceRoot');
isFunction(getFiles) || reportMissingOption('getFiles');
this.pkg = pkg;
@@ -36,6 +38,9 @@ export default class SourceArch {
// unibuild.
this.arch = arch;
// Absolute path of the root directory of this package or application.
this.sourceRoot = sourceRoot;
// Packages used. The ordering is significant only for determining
// import symbol priority (it doesn't affect load order), and a given
// package could appear more than once in the list, so code that
@@ -88,6 +93,13 @@ export default class SourceArch {
// this package) to compute this.
this.getFiles = getFiles;
// Object whose keys are relative paths of local node_modules
// directories in this package or application, for the given
// architecture. Does not include the .npm/package/node_modules
// directory installed by Npm.depends. Should be populated when
// getFiles is called.
this.localNodeModulesDirs = Object.create(null);
// Symbols that this architecture should export. List of symbols (as
// strings).
this.declaredExports = declaredExports;

42
tools/static-assets/server/boot.js
View File

@@ -139,6 +139,23 @@ var startCheckForLiveParent = function (parentPid) {
Fiber(function () {
_.each(serverJson.load, function (fileInfo) {
var code = fs.readFileSync(path.resolve(serverDir, fileInfo.path));
var nonLocalNodeModulesPaths = [];
function addNodeModulesPath(path) {
nonLocalNodeModulesPaths.push(
files.pathResolve(serverDir, path)
);
}
if (typeof fileInfo.node_modules === "string") {
addNodeModulesPath(fileInfo.node_modules);
} else if (fileInfo.node_modules) {
_.each(fileInfo.node_modules, function (info, path) {
if (! info.local) {
addNodeModulesPath(path);
}
});
}
var Npm = {
/**
@@ -149,20 +166,25 @@ Fiber(function () {
* @memberOf Npm
*/
require: function (name) {
if (! fileInfo.node_modules) {
if (nonLocalNodeModulesPaths.length === 0) {
return require(name);
}
var nodeModuleBase = path.resolve(serverDir,
files.convertToOSPath(fileInfo.node_modules));
var nodeModuleDir = path.resolve(nodeModuleBase, name);
var fullPath;
// If the user does `Npm.require('foo/bar')`, then we should resolve to
// the package's node modules if `foo` was one of the modules we
// installed. (`foo/bar` might be implemented as `foo/bar.js` so we
// can't just naively see if all of nodeModuleDir exists.
if (fs.existsSync(path.resolve(nodeModuleBase, name.split("/")[0]))) {
return require(nodeModuleDir);
nonLocalNodeModulesPaths.some(function (nodeModuleBase) {
var packageBase = files.pathResolve(
nodeModuleBase,
name.split("/", 1)[0]
);
if (fs.existsSync(packageBase)) {
return fullPath = files.pathResolve(nodeModuleBase, name);
}
});
if (fullPath) {
return require(fullPath);
}
try {