Files
meteor/tools/meteor_npm.js

579 lines
22 KiB
JavaScript

/// Implements the process of managing a package's .npm directory,
/// in which we call `npm install` to install npm dependencies,
/// and a variety of related commands. Notably, we use `npm shrinkwrap`
/// to ensure we get consistent versions of npm sub-dependencies.
var semver = require('semver');
var Future = require('fibers/future');
var path = require('path');
var fs = require('fs');
var cleanup = require(path.join(__dirname, 'cleanup.js'));
var files = require(path.join(__dirname, 'files.js'));
var httpHelpers = require('./http-helpers.js');
var buildmessage = require('./buildmessage.js');
var _ = require('underscore');
// if a user exits meteor while we're trying to create a .npm
// directory, we will have temporary directories that we clean up
cleanup.onExit(function () {
_.each(meteorNpm._tmpDirs, function (dir) {
if (fs.existsSync(dir))
files.rm_recursive(dir);
});
});
// Exception used internally to gracefully bail out of a npm run if
// something goes wrong
var NpmFailure = function () {};
var meteorNpm = exports;
_.extend(exports, {
_tmpDirs: [],
_isGitHubTarball: function (x) {
return /^https:\/\/github.com\/.*\/tarball\/[0-9a-f]{40}/.test(x);
},
// If there is a version that isn't exact, throws an Error with a
// human-readable message that is suitable for showing to the user.
// npmDependencies may be falsey or empty.
ensureOnlyExactVersions: function(npmDependencies) {
var self = this;
_.each(npmDependencies, function(version, name) {
// We want a given version of a smart package (package.js +
// .npm/npm-shrinkwrap.json) to pin down its dependencies precisely, so we
// don't want anything too vague. For now, we support semvers and github
// tarballs pointing at an exact commit.
if (!semver.valid(version) && !self._isGitHubTarball(version))
throw new Error(
"Must declare exact version of npm package dependency: " + name + '@' + version);
});
},
// Creates a temporary directory in which the new contents of the package's
// .npm directory will be assembled. If all is successful, renames that
// directory back to .npm. Returns true if there are NPM dependencies and
// they are installed without error.
//
// @param npmDependencies {Object} dependencies that should be installed,
// eg {tar: '0.1.6', gcd: '0.0.0'}. If falsey or empty, will remove
// the .npm directory instead.
updateDependencies: function(packageName,
packageNpmDir,
npmDependencies,
quiet) {
var self = this;
// we make sure to put it beside the original package dir so that
// we can then atomically rename it. we also make sure to
// randomize the name, in case we're bundling this package
// multiple times in parallel.
var newPackageNpmDir = packageNpmDir + '-new-' + self._randomToken();
if (!npmDependencies || _.isEmpty(npmDependencies)) {
// No NPM dependencies? Delete the .npm directory if it exists (because,
// eg, we used to have NPM dependencies but don't any more). We'd like to
// do this in as atomic a way as possible in case multiple meteor
// instances are trying to make this update in parallel, so we rename the
// directory to something before doing the rm -rf.
try {
fs.renameSync(packageNpmDir, newPackageNpmDir);
} catch (e) {
if (e.code !== 'ENOENT')
throw e;
// It didn't exist, which is exactly what we wanted.
return false;
}
files.rm_recursive(newPackageNpmDir);
return false;
}
try {
// v0.6.0 had a bug that could cause .npm directories to be
// created without npm-shrinkwrap.json
// (https://github.com/meteor/meteor/pull/927). Running your app
// in that state causes consistent "Corrupted .npm directory"
// errors.
//
// If you've reached that state, delete the empty directory and
// proceed.
if (fs.existsSync(packageNpmDir) &&
!fs.existsSync(path.join(packageNpmDir, 'npm-shrinkwrap.json'))) {
files.rm_recursive(packageNpmDir);
}
if (fs.existsSync(packageNpmDir)) {
// we already nave a .npm directory. update it appropriately with some ceremony involving:
// `npm install`, `npm install name@version`, `npm shrinkwrap`
self._updateExistingNpmDirectory(
packageName, newPackageNpmDir, packageNpmDir, npmDependencies, quiet);
} else {
// create a fresh .npm directory with `npm install name@version` and `npm shrinkwrap`
self._createFreshNpmDirectory(
packageName, newPackageNpmDir, packageNpmDir, npmDependencies, quiet);
}
} catch (e) {
if (e instanceof NpmFailure) {
// Something happened that was out of our control, but wasn't
// exactly unexpected (eg, no such npm package, no internet
// connection.) Handle it gracefully.
return false;
}
// Some other exception -- let it propagate.
throw e;
} finally {
if (fs.existsSync(newPackageNpmDir))
files.rm_recursive(newPackageNpmDir);
self._tmpDirs = _.without(self._tmpDirs, newPackageNpmDir);
}
return true;
},
// Return true if all of a package's npm dependencies are portable
// (that is, if the node_modules can be copied anywhere and we'd
// expect it to work, rather than containing native extensions that
// were built just for our architecture), else
// false. updateDependencies should first be used to bring
// packageNpmDir up to date.
dependenciesArePortable: function (packageNpmDir) {
// We use a simple heuristic: we check to see if a package (or any
// of its transitive depedencies) contains any *.node files. .node
// is the extension that signals to Node that it should load a
// file as a shared object rather than as JavaScript, so this
// should work in the vast majority of cases.
var search = function (dir) {
return _.find(fs.readdirSync(dir), function (itemName) {
if (itemName.match(/\.node$/))
return true;
var item = path.join(dir, itemName);
if (fs.statSync(item).isDirectory())
return search(item);
}) || false;
};
return ! search(path.join(packageNpmDir, 'node_modules'));
},
_makeNewPackageNpmDir: function (newPackageNpmDir) {
var self = this;
self._tmpDirs.push(newPackageNpmDir); // keep track so that we can remove them on process exit
files.mkdir_p(newPackageNpmDir);
// create node_modules -- prevent npm install from installing
// to an existing node_modules dir higher up in the filesystem
fs.mkdirSync(path.join(newPackageNpmDir, 'node_modules'));
// create .gitignore -- node_modules shouldn't be in git since we
// recreate it as needed by using `npm install`. since we use `npm
// shrinkwrap` we're guaranteed to have the same version installed
// each time.
fs.writeFileSync(
path.join(newPackageNpmDir, '.gitignore'),
['node_modules', ''/*git diff complains without trailing newline*/].join('\n'));
},
_updateExistingNpmDirectory: function(
packageName, newPackageNpmDir, packageNpmDir, npmDependencies, quiet) {
var self = this;
// sanity check on contents of .npm directory
if (!fs.statSync(packageNpmDir).isDirectory())
throw new Error("Corrupted .npm directory -- should be a directory: " + packageNpmDir);
if (!fs.existsSync(path.join(packageNpmDir, 'npm-shrinkwrap.json')))
throw new Error(
"Corrupted .npm directory -- can't find npm-shrinkwrap.json in " + packageNpmDir);
// We need to rebuild all node modules when the Node version changes, in
// case there are some binary ones. Technically this is racey, but it
// shouldn't fail very often.
if (fs.existsSync(path.join(packageNpmDir, 'node_modules'))) {
var oldNodeVersion;
try {
oldNodeVersion = fs.readFileSync(
path.join(packageNpmDir, 'node_modules', '.node_version'), 'utf8');
} catch (e) {
if (e.code !== 'ENOENT')
throw e;
// Use the Node version from the last release where we didn't drop this
// file.
oldNodeVersion = 'v0.8.24';
}
if (oldNodeVersion !== process.version)
files.rm_recursive(path.join(packageNpmDir, 'node_modules'));
}
var installedDependencies = self._installedDependencies(packageNpmDir);
// If we already have the right things installed, life is good.
if (_.isEqual(installedDependencies, npmDependencies))
return;
if (!quiet)
self._logUpdateDependencies(packageName, npmDependencies);
var shrinkwrappedDependenciesTree =
self._shrinkwrappedDependenciesTree(packageNpmDir);
var shrinkwrappedDependencies = self._treeToDependencies(
shrinkwrappedDependenciesTree);
var preservedShrinkwrap = {dependencies: {}};
_.each(shrinkwrappedDependencies, function (version, name) {
if (npmDependencies[name] === version) {
// We're not changing this dependency, so copy over its shrinkwrap.
preservedShrinkwrap.dependencies[name] =
shrinkwrappedDependenciesTree.dependencies[name];
}
});
self._makeNewPackageNpmDir(newPackageNpmDir);
if (!_.isEmpty(preservedShrinkwrap.dependencies)) {
// There are some unchanged packages here. Install from shrinkwrap.
fs.writeFileSync(path.join(newPackageNpmDir, 'npm-shrinkwrap.json'),
JSON.stringify(preservedShrinkwrap, null, /*legible*/2));
// construct a matching package.json to make `npm install` happy
self._constructPackageJson(packageName, newPackageNpmDir,
self._treeToDependencies(preservedShrinkwrap));
// `npm install`
self._installFromShrinkwrap(newPackageNpmDir);
// delete package.json and npm-shrinkwrap.json
fs.unlinkSync(path.join(newPackageNpmDir, 'package.json'));
fs.unlinkSync(path.join(newPackageNpmDir, 'npm-shrinkwrap.json'));
}
// we may have just installed the shrinkwrapped packages. but let's not
// trust that it actually worked: let's do the rest based on what we
// actually have installed now.
var newInstalledDependencies = self._installedDependencies(newPackageNpmDir);
// `npm install name@version` for modules that need updating
_.each(npmDependencies, function(version, name) {
if (newInstalledDependencies[name] !== version) {
self._installNpmModule(name, version, newPackageNpmDir);
}
});
self._completeNpmDirectory(
packageName, newPackageNpmDir, packageNpmDir, npmDependencies);
},
_createFreshNpmDirectory: function(
packageName, newPackageNpmDir, packageNpmDir, npmDependencies, quiet) {
var self = this;
if (!quiet)
self._logUpdateDependencies(packageName, npmDependencies);
self._makeNewPackageNpmDir(newPackageNpmDir);
// install dependencies
_.each(npmDependencies, function(version, name) {
self._installNpmModule(name, version, newPackageNpmDir);
});
self._completeNpmDirectory(
packageName, newPackageNpmDir, packageNpmDir, npmDependencies);
},
// Shared code for _updateExistingNpmDirectory and _createFreshNpmDirectory.
_completeNpmDirectory: function (
packageName, newPackageNpmDir, packageNpmDir, npmDependencies) {
var self = this;
// temporarily construct a matching package.json to make `npm shrinkwrap`
// happy
self._constructPackageJson(packageName, newPackageNpmDir, npmDependencies);
// Create a shrinkwrap file.
self._shrinkwrap(newPackageNpmDir);
// now delete package.json
fs.unlinkSync(path.join(newPackageNpmDir, 'package.json'));
self._createReadme(newPackageNpmDir);
self._createNodeVersion(newPackageNpmDir);
files.renameDirAlmostAtomically(newPackageNpmDir, packageNpmDir);
},
_createReadme: function(newPackageNpmDir) {
fs.writeFileSync(
path.join(newPackageNpmDir, 'README'),
"This directory and the files immediately inside it are automatically generated\n"
+ "when you change this package's NPM dependencies. Commit the files in this\n"
+ "directory (npm-shrinkwrap.json, .gitignore, and this README) to source control\n"
+ "so that others run the same versions of sub-dependencies.\n"
+ "\n"
+ "You should NOT check in the node_modules directory that Meteor automatically\n"
+ "creates; if you are using git, the .gitignore file tells git to ignore it.\n"
);
},
_createNodeVersion: function(newPackageNpmDir) {
fs.writeFileSync(
path.join(newPackageNpmDir, 'node_modules', '.node_version'),
process.version);
},
// Returns object with keys 'stdout', 'stderr', and 'success' (true
// for clean exit with exit code 0, else false)
_execFileSync: function(file, args, opts) {
var self = this;
if (self._printNpmCalls) // only used by test_bundler.js
process.stdout.write('cd ' + opts.cwd + ' && ' + file + ' ' + args.join(' ') + ' ... ');
var future = new Future;
var child_process = require('child_process');
child_process.execFile(file, args, opts, function (err, stdout, stderr) {
if (self._printNpmCalls)
console.log(err ? 'failed' : 'done');
future.return({
success: ! err,
stdout: stdout,
stderr: stderr
});
});
return future.wait();
},
_constructPackageJson: function(packageName, newPackageNpmDir, npmDependencies) {
var packageJsonContents = JSON.stringify({
// name and version are unimportant but required for `npm install`
name: 'packages-for-meteor-smartpackage-' + packageName,
version: '0.0.0',
dependencies: npmDependencies
});
var packageJsonPath = path.join(newPackageNpmDir, 'package.json');
fs.writeFileSync(packageJsonPath, packageJsonContents);
},
// Gets a JSON object from `npm ls --json` (_installedDependenciesTree) or
// `npm-shrinkwrap.json` (_shrinkwrappedDependenciesTree).
//
// @returns {Object} eg {
// "name": "packages",
// "version": "0.0.0",
// "dependencies": {
// "sockjs": {
// "version": "0.3.4",
// "dependencies": {
// "node-uuid": {
// "version": "1.3.3"
// }
// }
// }
// }
// }
_installedDependenciesTree: function(dir) {
var result =
this._execFileSync(path.join(files.get_dev_bundle(), "bin", "npm"),
["ls", "--json"],
{cwd: dir});
if (result.success)
return JSON.parse(result.stdout);
console.log(result.stderr);
buildmessage.error("couldn't read npm version lock information");
// Recover by returning false from updateDependencies
throw new NpmFailure;
},
_shrinkwrappedDependenciesTree: function(dir) {
var shrinkwrapFile = fs.readFileSync(path.join(dir, 'npm-shrinkwrap.json'));
return JSON.parse(shrinkwrapFile);
},
// Maps a "dependency object" (a thing you find in `npm ls --json` or
// npm-shrinkwrap.json with keys like "version" and "from") to the canonical
// version that matches what users put in the `Npm.depends` clause. ie,
// either the version or the tarball URL.
// If more logic is added here, it should probably go in minimizeModule too.
_canonicalVersion: function (depObj) {
var self = this;
if (self._isGitHubTarball(depObj.from))
return depObj.from;
else
return depObj.version;
},
// map the structure returned from `npm ls` or shrinkwrap.json into the
// structure of npmDependencies (e.g. {gcd: '0.0.0'}), so that they can be
// diffed. This only returns top-level dependencies.
_treeToDependencies: function (tree) {
var self = this;
return _.object(
_.map(
tree.dependencies, function(properties, name) {
return [name, self._canonicalVersion(properties)];
}));
},
_installedDependencies: function(dir) {
var self = this;
return self._treeToDependencies(self._installedDependenciesTree(dir));
},
_shrinkwrappedDependencies: function (dir) {
var self = this;
return self._treeToDependencies(self._shrinkwrappedDependenciesTree(dir));
},
_installNpmModule: function(name, version, dir) {
this._ensureConnected();
var installArg = this._isGitHubTarball(version)
? version : (name + "@" + version);
// We don't use npm.commands.install since we couldn't
// figure out how to silence all output (specifically the
// installed tree which is printed out with `console.log`)
//
// We use --force, because the NPM cache is broken! See
// https://github.com/isaacs/npm/issues/3265 Basically, switching back and
// forth between a tarball fork of version X and the real version X can
// confuse NPM. But the main reason to use tarball URLs is to get a fork of
// the latest version with some fix, so it's easy to trigger this! So
// instead, always use --force. (Even with --force, we still WRITE to the
// cache, so we can corrupt the cache for other invocations of npm... ah
// well.)
var result =
this._execFileSync(path.join(files.get_dev_bundle(), "bin", "npm"),
["install", "--force", installArg],
{cwd: dir});
if (! result.success) {
var pkgNotFound = "404 '" + name + "' is not in the npm registry";
var versionNotFound = "version not found: " + version;
if (result.stderr.match(new RegExp(pkgNotFound))) {
buildmessage.error("there is no npm package named '" + name + "'");
} else if (result.stderr.match(new RegExp(versionNotFound))) {
buildmessage.error(name + " version " + version + " " +
"is not available in the npm registry");
} else {
console.log(result.stderr);
buildmessage.error("couldn't install npm package");
}
// Recover by returning false from updateDependencies
throw new NpmFailure;
}
},
_installFromShrinkwrap: function(dir) {
if (!fs.existsSync(path.join(dir, "npm-shrinkwrap.json")))
throw new Error("Can't call `npm install` without a npm-shrinkwrap.json file present");
this._ensureConnected();
// `npm install`, which reads npm-shrinkwrap.json. See above for why
// --force.
var result =
this._execFileSync(path.join(files.get_dev_bundle(), "bin", "npm"),
["install", "--force"], {cwd: dir});
if (! result.success) {
console.log(result.stderr);
buildmessage.error("couldn't install npm packages from npm-shrinkwrap");
// Recover by returning false from updateDependencies
throw new NpmFailure;
}
},
// ensure we can reach http://npmjs.org before we try to install
// dependencies. `npm install` times out after more than a minute.
_ensureConnected: function () {
try {
httpHelpers.getUrl("http://registry.npmjs.org");
} catch (e) {
buildmessage.error("Can't install npm dependencies. " +
"Are you connected to the internet?");
// Recover by returning false from updateDependencies
throw new NpmFailure;
}
},
// `npm shrinkwrap`
_shrinkwrap: function(dir) {
var self = this;
// We don't use npm.commands.shrinkwrap for two reasons:
// 1. As far as we could tell there's no way to completely silence the output
// (the `silent` flag isn't piped in to the call to npm.commands.ls)
// 2. In various (non-deterministic?) cases we observed the
// npm-shrinkwrap.json file not being updated
var result =
this._execFileSync(path.join(files.get_dev_bundle(), "bin", "npm"),
["shrinkwrap"], {cwd: dir});
if (! result.success) {
console.log(result.stderr);
buildmessage.error("couldn't run `npm shrinkwrap`");
// Recover by returning false from updateDependencies
throw new NpmFailure;
}
self._minimizeShrinkwrap(dir);
},
// The shrinkwrap file format contains a lot of extra data that can change as
// you re-run the NPM-update process without actually affecting what is
// installed. This step trims everything but the most important bits from the
// file, so that the file doesn't change unnecessary.
//
// This is based on an analysis of install.js in the npm module:
// https://github.com/isaacs/npm/blob/master/lib/install.js
// It appears that the only things actually read from a given dependency are
// its sub-dependencies and a single version, which is read by the readWrap
// function; and furthermore, we can just put all versions in the "version"
// field.
_minimizeShrinkwrap: function (dir) {
var self = this;
var topLevel = self._shrinkwrappedDependenciesTree(dir);
var minimizeModule = function (module) {
var minimized = {};
if (self._isGitHubTarball(module.from))
minimized.from = module.from;
else
minimized.version = module.version;
if (module.dependencies) {
minimized.dependencies = {};
_.each(module.dependencies, function (subModule, name) {
minimized.dependencies[name] = minimizeModule(subModule);
});
}
return minimized;
};
var newTopLevelDependencies = {};
_.each(topLevel.dependencies, function (module, name) {
newTopLevelDependencies[name] = minimizeModule(module);
});
fs.writeFileSync(
path.join(dir, 'npm-shrinkwrap.json'),
// Matches the formatting done by 'npm shrinkwrap'.
JSON.stringify({dependencies: newTopLevelDependencies}, null, 2)
+ '\n');
},
_logUpdateDependencies: function(packageName, npmDependencies) {
console.log('%s: updating npm dependencies -- %s...',
packageName, _.keys(npmDependencies).join(', '));
},
_randomToken: function() {
return (Math.random() * 0x100000000 + 1).toString(36);
}
});