untargz in process when fetching packages

lib/files.js

@@ -4,6 +4,7 @@ var _ = require('underscore');
 var zlib = require("zlib");
 var tar = require("tar");
 var Future = require('fibers/future');
+var request = require('request');
 
 var files = module.exports = {
   // A sort comparator to order files into load order.
@@ -372,6 +373,14 @@ var files = module.exports = {
       throw error;
 
     // succeed! no return value.
+  },
+
+  // A thin wrapper around request(...) that makes the response "body"
+  // the main callback argument.  This facilitates using `Future.wrap`.
+  getUrl: function (urlOrOptions, callback) {
+    return request(urlOrOptions, function (error, response, body) {
+      callback(error, body, response);
+    });
   }
 
 };

lib/packages.js

@@ -412,7 +412,11 @@ var packages = module.exports = {
   },
 
   existsInPackageCache: function (name, version) {
-    return fs.existsSync(path.join(__dirname, '..', 'cache', 'packages', name, version));
+    // Look for presence of "package.js" file in directory so we don't count
+    // an empty dir as a package.  An empty dir could be left by a failed
+    // package untarring, for example.
+    return fs.existsSync(path.join(__dirname, '..', 'cache', 'packages', name, version,
+                                   'package.js'));
   },
 
   // fetches the manifest file for the given release version. also fetches
@@ -477,27 +481,27 @@ var packages = module.exports = {
       _.each(manifest.packages, function (version, name) {
         if (!self.existsInPackageCache(name, version)) {
           var packageDir = path.join(__dirname, '..', 'cache', 'packages', name, version);
-          files.mkdir_p(packageDir);
+          var packageUrl = PACKAGES_URLBASE + "/packages/" + name + "/" +
+                version + ".tar.gz";
 
-          var future = new Future;
-          futures.push(future);
-          exec(
-            "curl " + PACKAGES_URLBASE + "/packages/" + name + "/"
-              + version + ".tar.gz"
-              + "| tar xvz",
-            {cwd: packageDir},
-            function (error, stdout, stderr) {
-              console.log('stdout: ' + stdout);
-              console.log('stderr: ' + stderr);
-              if (error !== null) {
-                console.log('exec error: ' + error);
-              }
-              future['return']();
+          console.log("Fetching " + packageUrl + "...");
+          futures.push(Future.wrap(function (cb) {
+            files.getUrl({url: packageUrl, encoding: null}, function (error, result) {
+              if (! error && result)
+                result = { buffer: result, packageDir: packageDir };
+              cb(error, result);
             });
-          }
+          })());
+        }
       });
 
       Future.wait(futures);
+
+      _.each(futures, function (f) {
+        var result = f.get();
+        files.mkdir_p(result.packageDir);
+        files.extractTarGz(result.buffer, result.packageDir);
+      });
     }
   }
 };