diff --git a/lib/files.js b/lib/files.js index 7374f2e232..77f233161a 100644 --- a/lib/files.js +++ b/lib/files.js @@ -112,14 +112,17 @@ var files = module.exports = { // XXX once we are done with the transition to engine, this should // change to: `return fs.existsSync(path.join(filepath, '.meteor', // 'release'))` - var testFilePath = path.join(filepath, '.meteor', 'packages'); // .meteor/packages can be a directory, if .meteor is a warehouse // directory. since installing meteor initializes a warehouse at // $HOME/.meteor, we want to make sure your home directory (and // all subdirectories therein) don't count as being within a // meteor app. - return fs.existsSync(testFilePath) && fs.statSync(testFilePath).isFile(); + try { // use try/catch to avoid the additional syscall to fs.existsSync + return fs.statSync(path.join(filepath, '.meteor', 'packages')).isFile(); + } catch (e) { + return false; + } }, // given a path, returns true if it is a meteor package (is a diff --git a/lib/packages.js b/lib/packages.js index b2e2cc4170..66b9fd7ef4 100644 --- a/lib/packages.js +++ b/lib/packages.js @@ -1,6 +1,7 @@ var path = require('path'); var _ = require('underscore'); var files = require(path.join(__dirname, 'files.js')); +var warehouse = require(path.join(__dirname, 'warehouse.js')); var meteorNpm = require(path.join(__dirname, 'meteor_npm.js')); var fs = require('fs'); @@ -139,7 +140,7 @@ _.extend(Package.prototype, { initFromWarehouse: function (name, version) { this._initFromPackageDir( name, - path.join(files.getWarehouseDir(), 'packages', name, version)); + path.join(warehouse.getWarehouseDir(), 'packages', name, version)); }, init_from_app_dir: function (app_dir, ignore_files) { @@ -393,7 +394,9 @@ var packages = module.exports = { }, _localPackageDirs: function () { - var packageDirs = [path.join(files.get_core_dir(), 'packages')]; // only for a checkout + var packageDirs = []; + if (files.in_checkout()) + packageDirs.push(path.join(files.get_core_dir(), 'packages')); if (process.env.PACKAGE_DIRS) packageDirs = process.env.PACKAGE_DIRS.split(':').concat(packageDirs); diff --git a/lib/warehouse.js b/lib/warehouse.js index dbc4444663..533361a18e 100644 --- a/lib/warehouse.js +++ b/lib/warehouse.js @@ -1,9 +1,11 @@ -/// We store a "warehouse" of engine, releases and packages on +/// We store a "warehouse" of engines, releases and packages on /// disk. This warehouse is populated from our servers, as needed. /// /// Directory structure: /// +/// meteor (relative path symlink to engines/latest/bin/meteor) /// engines/ (not in checkout, since we run against checked-out code) +/// latest/ (relative path symlink to latest x.y.z/ engine directory) /// x.y.z/ /// releases/ /// x.y.z.json @@ -27,9 +29,7 @@ var warehouse = module.exports = { // Return our loaded collection of engines, releases and // packages. If we're running an installed version, found at // $HOME/.meteor. If we're running a checkout, found at - // $CHECKOUT/.meteor. This directory has the following subdirectory - // structure: - // + // $CHECKOUT/.meteor. getWarehouseDir: function () { if (files.in_checkout()) return path.join(files.get_core_dir(), '.meteor'); @@ -37,19 +37,12 @@ var warehouse = module.exports = { return path.join(process.env.HOME, '.meteor'); }, - // Load the manifest corresponding to a given meteor release from - // packages.meteor.com and store in the warehouse, on disk. Parse - // and ensure that all used package versions are also stored in the - // warehouse. Return parsed manifest. - // Load and return a manifest for an app, based on the - // .meteor/release file - // // If .meteor/release exists, load the manifest corresponding to // that meteor release. Load from packages.meteor.com and store in // the warehouse on disk. Parse and ensure that all used package // versions are stored. Return parsed manifest. // - // If .meteor/version does not exist, return null. + // If .meteor/release does not exist, return null. manifestForApp: function (appDir) { var releaseVersion = project.getMeteorReleaseVersion(appDir); @@ -72,62 +65,12 @@ var warehouse = module.exports = { releaseManifest = JSON.parse(fs.readFileSync(releaseManifestPath)); } else { // grow warehouse with new manifest and packages - releaseManifest = warehouse.populateWarehouseForRelease(releaseVersion); + releaseManifest = warehouse._populateWarehouseForRelease(releaseVersion); } - var Future = require('fibers/future'); - var futures = []; - _.each(releaseManifest.packages, function (version, name) { - if (!warehouse.existsInWarehouse(name, version)) { - var packageDir = path.join(warehouse.getWarehouseDir(), 'packages', name, version); - var packageUrl = PACKAGES_URLBASE + "/packages/" + name + "/" + - name + '-' + version + ".tar.gz"; - - console.log("Fetching " + packageUrl + "..."); - futures.push(Future.wrap(function (cb) { - files.getUrl({url: packageUrl, encoding: null}, function (error, result) { - if (! error && result) - result = { buffer: result, packageDir: packageDir }; - cb(error, result); - }); - })()); - } - }); - - Future.wait(futures); - - _.each(futures, function (f) { - var result = f.get(); - files.mkdir_p(result.packageDir); - files.extractTarGz(result.buffer, result.packageDir); - }); - return releaseManifest; }, - // fetches the manifest file for the given release version. also fetches - // all of the missing versioned packages referenced from the manifest - // @param releaseVersion {String} eg "0.1" - // @returns {Object} parsed manifest file - populateWarehouseForRelease: function(releaseVersion) { - var future = new Future; - var releasesDir = path.join(warehouse.getWarehouseDir(), 'releases'); - files.mkdir_p(releasesDir, 0755); - var releaseManifestPath = path.join(releasesDir, releaseVersion + '.json'); - - // load the manifest from s3, and store in the warehouse - try { - var releaseManifest = Future.wrap(files.getUrl)( - PACKAGES_URLBASE + "/manifest/" + releaseVersion + ".json").wait(); - fs.writeFileSync(releaseManifestPath, releaseManifest); - return JSON.parse(manifest); - } catch (e) { - console.error( - "Can't find manifest for meteor release version " + releaseVersion); - throw e; - } - }, - // look in the warehouse for the latest release version latestRelease: function() { var releasesDir = path.join(warehouse.getWarehouseDir(), 'releases'); @@ -153,6 +96,74 @@ var warehouse = module.exports = { // package untarring, for example. return fs.existsSync( path.join(warehouse.getWarehouseDir(), 'packages', name, version, 'package.js')); - } + }, + // fetches the manifest file for the given release version. also fetches + // all of the missing versioned packages referenced from the manifest + // @param releaseVersion {String} eg "0.1" + // @returns {Object} parsed manifest file + _populateWarehouseForRelease: function(releaseVersion) { + var future = new Future; + var releasesDir = path.join(warehouse.getWarehouseDir(), 'releases'); + files.mkdir_p(releasesDir, 0755); + var releaseManifestPath = path.join(releasesDir, releaseVersion + '.json'); + + try { + // get release manifest, but only write it after we're done + // writing packages + var releaseManifest = JSON.parse(Future.wrap(files.getUrl)( + PACKAGES_URLBASE + "/manifest/" + releaseVersion + ".json").wait()); + + // populate warehouse with missing packages + var missingPackages = {}; + _.each(releaseManifest.packages, function (version, name) { + if (!warehouse.existsInWarehouse(name, version)) { + missingPackages[name] = version; + } + }); + warehouse._populateWarehouseWithPackages(missingPackages); + + // now that we have written all packages, it's safe to write the + // release manifest + fs.writeFileSync(releaseManifestPath, JSON.stringify(releaseManifest)); + + // return manifest + return releaseManifest; + } catch (e) { + console.error( + "Can't find manifest for meteor release version " + releaseVersion); + throw e; + } + }, + + // @param packages {Object} eg {"less": "0.5.0"} + _populateWarehouseWithPackages: function(packages) { + var Future = require('fibers/future'); + var futures = []; + _.each(packages, function (version, name) { + var packageDir = path.join(warehouse.getWarehouseDir(), 'packages', name, version); + var packageUrl = PACKAGES_URLBASE + "/packages/" + name + "/" + + name + '-' + version + ".tar.gz"; + + console.log("Fetching " + packageUrl + "..."); + futures.push(Future.wrap(function (cb) { + files.getUrl({url: packageUrl, encoding: null}, function (error, result) { + if (! error && result) + result = { buffer: result, packageDir: packageDir }; + cb(error, result); + }); + })()); + }); + + Future.wait(futures); + + _.each(futures, function (f) { + var result = f.get(); + // extract to a temporary directory and then rename, to ensure + // we don't end up with a corrupt warehouse + files.mkdir_p(result.packageDir + ".tmp"); + files.extractTarGz(result.buffer, result.packageDir + ".tmp"); + fs.renameSync(result.packageDir + ".tmp", result.packageDir); + }); + } };