From 4aa1201d9f620e569718519dcc0dcba3893365fe Mon Sep 17 00:00:00 2001
From: Avital Oliver <avital@thewe.net>
Date: Wed, 20 Feb 2013 14:32:31 -0800
Subject: [PATCH] Publish release

---
 lib/warehouse.js                              |  33 ++--
 tools/admin/build-package-tarballs.sh         |  11 +-
 tools/admin/publish-release-packages/README   |   2 +
 tools/admin/publish-release-packages/awssum   |   1 +
 tools/admin/publish-release.sh                |  25 +++
 .../admin/publish-release/.meteor/.gitignore  |   1 +
 tools/admin/publish-release/.meteor/packages  |   9 +
 .../publish-release/server/publish-release.js | 167 ++++++++++++++++++
 8 files changed, 234 insertions(+), 15 deletions(-)
 create mode 100644 tools/admin/publish-release-packages/README
 create mode 160000 tools/admin/publish-release-packages/awssum
 create mode 100755 tools/admin/publish-release.sh
 create mode 100644 tools/admin/publish-release/.meteor/.gitignore
 create mode 100644 tools/admin/publish-release/.meteor/packages
 create mode 100644 tools/admin/publish-release/server/publish-release.js

diff --git a/lib/warehouse.js b/lib/warehouse.js
index f4b997b330..154c32bb8d 100644
--- a/lib/warehouse.js
+++ b/lib/warehouse.js
@@ -116,13 +116,19 @@ var warehouse = module.exports = {
     files.mkdir_p(releasesDir, 0755);
     var releaseManifestPath = path.join(releasesDir, releaseVersion + '.json');
 
+    // get release manifest, but only write it after we're done
+    // writing packages
+    var releaseManifest;
     try {
-      // get release manifest, but only write it after we're done
-      // writing packages
-      var releaseManifest = JSON.parse(Future.wrap(files.getUrl)(
-        PACKAGES_URLBASE + "/manifest/" + releaseVersion + ".json").wait());
+      releaseManifest = JSON.parse(Future.wrap(files.getUrl)(
+        PACKAGES_URLBASE + "/releases/" + releaseVersion + ".json").wait());
+    } catch (e) {
+      console.error("Can't find release " + releaseVersion);
+      process.exit(1);
+    }
 
-      // populate warehouse with missing packages
+    // populate warehouse with missing packages
+    try {
       var missingPackages = {};
       _.each(releaseManifest.packages, function (version, name) {
         if (!warehouse.existsInWarehouse(name, version)) {
@@ -130,18 +136,17 @@ var warehouse = module.exports = {
         }
       });
       warehouse._populateWarehouseWithPackages(missingPackages);
-
-      // now that we have written all packages, it's safe to write the
-      // release manifest
-      fs.writeFileSync(releaseManifestPath, JSON.stringify(releaseManifest));
-
-      // return manifest
-      return releaseManifest;
     } catch (e) {
-      console.error(
-        "Can't find manifest for meteor release version " + releaseVersion);
+      console.error("Failed to load packages for release " + releaseVersion);
       throw e;
     }
+
+    // now that we have written all packages, it's safe to write the
+    // release manifest
+    fs.writeFileSync(releaseManifestPath, JSON.stringify(releaseManifest));
+
+    // return manifest
+    return releaseManifest;
   },
 
   // @param packagesToPopulate {Object} eg {"less": "0.5.0"}
diff --git a/tools/admin/build-package-tarballs.sh b/tools/admin/build-package-tarballs.sh
index 0b00c066c9..f5b222d178 100755
--- a/tools/admin/build-package-tarballs.sh
+++ b/tools/admin/build-package-tarballs.sh
@@ -16,10 +16,15 @@ if [ -e "$TOPDIR/.package_manifest_chunk" ]; then
   rm "$TOPDIR/.package_manifest_chunk"
 fi
 
+FIRST_RUN=true # keep track to place commas correctly
 cd packages
 for PACKAGE in `ls`
 do
   if [ -a "$PACKAGE/package.js" ]; then
+    if [ $FIRST_RUN == false ]; then
+      echo "," >> "$TOPDIR/.package_manifest_chunk"
+    fi
+
     cd $PACKAGE
     PACKAGE_VERSION=$($TOPDIR/tools/admin/hash-dir.sh)
     echo "$PACKAGE version $PACKAGE_VERSION"
@@ -27,6 +32,10 @@ do
     cd ..
 
     # this is used in build-release.sh, which constructs the manifest json.
-    echo "    \"$PACKAGE\": \"$PACKAGE_VERSION\"" >> "$TOPDIR/.package_manifest_chunk"
+    echo -n "    \"$PACKAGE\": \"$PACKAGE_VERSION\"" >> "$TOPDIR/.package_manifest_chunk"
+    FIRST_RUN=false
   fi
 done
+
+# Add one newline at the end
+echo >> "$TOPDIR/.package_manifest_chunk"
diff --git a/tools/admin/publish-release-packages/README b/tools/admin/publish-release-packages/README
new file mode 100644
index 0000000000..8f0d718eaf
--- /dev/null
+++ b/tools/admin/publish-release-packages/README
@@ -0,0 +1,2 @@
+To generate:
+    git clone https://github.com/avital/meteor-awssum awssum
\ No newline at end of file
diff --git a/tools/admin/publish-release-packages/awssum b/tools/admin/publish-release-packages/awssum
new file mode 160000
index 0000000000..e8781acbfb
--- /dev/null
+++ b/tools/admin/publish-release-packages/awssum
@@ -0,0 +1 @@
+Subproject commit e8781acbfb616fb5c5972993e4769addfacacbbe
diff --git a/tools/admin/publish-release.sh b/tools/admin/publish-release.sh
new file mode 100755
index 0000000000..8ab268abeb
--- /dev/null
+++ b/tools/admin/publish-release.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+DIR="$(pwd)"
+METEOR_DIR="$(pwd)/../.."
+
+# publish-release is a meteor app
+cd publish-release
+
+# prepare settings file with git sha of last commit
+TMPDIR=$(mktemp -d -t meteor-publish-release-XXXXXXXX)
+GIT_SHA=$(git rev-parse HEAD)
+cat > "$TMPDIR/settings.json" <<EOF
+{"git-sha": "$GIT_SHA"}
+EOF
+
+# Run meteor with our awssum smart package.
+#
+# XXX when we support apps in packages fold this into the app itself.
+#
+# XXX when we support third-party packages use that mechanism instead
+# of keeping the pacakge in git.
+#
+# XXX it would be cool to be able to not listen on any port here. instead
+# we use port 31337
+PACKAGE_DIRS=$DIR/publish-release-packages $METEOR_DIR/meteor -p 31337 --once --settings=$TMPDIR/settings.json
+
diff --git a/tools/admin/publish-release/.meteor/.gitignore b/tools/admin/publish-release/.meteor/.gitignore
new file mode 100644
index 0000000000..4083037423
--- /dev/null
+++ b/tools/admin/publish-release/.meteor/.gitignore
@@ -0,0 +1 @@
+local
diff --git a/tools/admin/publish-release/.meteor/packages b/tools/admin/publish-release/.meteor/packages
new file mode 100644
index 0000000000..74931fd95a
--- /dev/null
+++ b/tools/admin/publish-release/.meteor/packages
@@ -0,0 +1,9 @@
+# Meteor packages used by this project, one per line.
+#
+# 'meteor add' and 'meteor remove' will edit this file for you,
+# but you can also edit it by hand.
+
+autopublish
+insecure
+preserve-inputs
+awssum
diff --git a/tools/admin/publish-release/server/publish-release.js b/tools/admin/publish-release/server/publish-release.js
new file mode 100644
index 0000000000..448e1463df
--- /dev/null
+++ b/tools/admin/publish-release/server/publish-release.js
@@ -0,0 +1,167 @@
+var amazon = AWSSum.load('amazon/amazon');
+var S3 = AWSSum.load('amazon/s3', 'S3');
+var Future = Npm.require("fibers/future");
+var child_process = Npm.require("child_process");
+
+// gets git sha passed in via --settings
+var getGitSha = function () {
+  var gitSha = Meteor.settings["git-sha"];
+  if (!gitSha) {
+    console.log("Run with --settings to set 'git-sha'");
+    process.exit(1);
+  }
+  return gitSha;
+};
+
+// calls 'cmd', returns stdout.
+// XXX should we have a smart package for these? 'process'?
+var execSync = function (cmd) {
+  return Future.wrap(function(cb) {
+    var cb2 = function(err, stdout, stderr) { cb(err, stdout); };
+    child_process.exec(cmd, cb2);
+  })().wait();
+};
+
+// returns usable S3 object
+var configureS3 = function () {
+  // calls 's3cmd --dump-config', returns {accessKey: ..., secretKey: ...}
+  var getS3Credentials = function () {
+    var unparsedConfig = execSync("s3cmd --dump-config");
+    var accessKey = /access_key = (.*)/.exec(unparsedConfig)[1];
+    var secretKey = /secret_key = (.*)/.exec(unparsedConfig)[1];
+    return {accessKey: accessKey, secretKey: secretKey};
+  };
+
+  var s3Credentials = getS3Credentials();
+  var s3 = new S3({
+    accessKeyId: s3Credentials.accessKey,
+    secretAccessKey: s3Credentials.secretKey,
+    region: amazon.US_EAST_1
+  });
+
+  return s3;
+};
+
+// fetch and parse release manifest
+var getManifest = function(s3, release) {
+  var content;
+  try {
+    content = s3.GetObject({
+      BucketName: "com.meteor.packages",
+      ObjectName: ["unpublished", release, "manifest.json"].join("/")
+    }).Body;
+  } catch (e) {
+    console.log("Release " + release + " not built.");
+    process.exit(1);
+  }
+
+  return JSON.parse(content);
+};
+
+// are there any files with this prefix?
+var notEmpty = function(s3, prefix) {
+  var files = s3.ListObjects({
+    BucketName: "com.meteor.packages",
+    Prefix: prefix
+  });
+  return !_.isEmpty(files.Body.ListBucketResult.Contents);
+};
+
+// publish a given engine, copying multiple files from
+// s3://com.meteor.packages/RELEASE/unpublished/ to
+// s3://com.meteor.packages/engines/VERSION/
+var publishEngine = function(s3, release, version) {
+  var destPath = ["engines", version].join("/");
+
+  process.stdout.write("Engine " + version + ": ");
+  if (notEmpty(s3, destPath)) {
+    console.log("already published");
+    return;
+  } else {
+    console.log("publishing");
+  }
+
+  var engineArtifacts = s3.ListObjects({
+    BucketName: "com.meteor.packages",
+    Prefix: ["unpublished", release, "meteor-engine-"].join("/")
+  }).Body.ListBucketResult.Contents;
+
+  _.each(engineArtifacts, function (artifact) {
+    var sourceKey = artifact.Key;
+    var filename = _.last(sourceKey.split("/"));
+    var destKey = [destPath, filename].join("/");
+
+    var opts = {
+      BucketName: "com.meteor.packages",
+      ObjectName: destKey,
+      SourceBucket: "com.meteor.packages",
+      SourceObject: sourceKey
+    };
+    s3.CopyObject(opts);
+  });
+};
+
+// publish a given package, copying from
+// s3://com.meteor.packages/unpublished/RELEASE/NAME-VERSION.tar.gz to
+// s3://com.meteor.packages/packages/NAME-VERSION.tar.gz
+var publishPackage = function(s3, release, name, version) {
+  var filename = name + "-" + version + ".tar.gz";
+  var destKey = ["packages", name, filename].join("/");
+  var sourceKey = ["unpublished", release, filename].join("/");
+
+  process.stdout.write("Package " + name + " version " + version + ": ");
+  if (notEmpty(s3, destKey)) {
+    console.log("already published");
+    return;
+  } else {
+    console.log("publishing");
+  }
+
+  var opts = {
+    BucketName: "com.meteor.packages",
+    ObjectName: destKey,
+    SourceBucket: "com.meteor.packages",
+    SourceObject: sourceKey
+  };
+  s3.CopyObject(opts);
+};
+
+// publish the release manifest, copying from
+// s3://com.meteor.packages/unpublished/RELEASE/manifest.json to
+// s3://com.meteor.packages/releases/RELEASE.json
+var publishManifest = function(s3, release) {
+  var destKey = ["releases", release + ".json"].join("/");
+  var sourceKey = ["unpublished", release, "manifest.json"].join("/");
+
+  process.stdout.write("Release manifest " + release + ": ");
+  if (notEmpty(s3, destKey)) {
+    console.log("already published");
+    return;
+  } else {
+    console.log("publishing");
+  }
+
+  var opts = {
+    BucketName: "com.meteor.packages",
+    ObjectName: destKey,
+    SourceBucket: "com.meteor.packages",
+    SourceObject: sourceKey
+  };
+  s3.CopyObject(opts);
+};
+
+// START HERE
+var main = function() {
+  // read git sha, used as the version of the unpublished release.
+  var release = getGitSha();
+  var s3 = configureS3();
+  var manifest = getManifest(s3, release);
+  publishEngine(s3, release, manifest.engine);
+  _.each(manifest.packages, function(version, name) {
+    publishPackage(s3, release, name, version);
+  });
+  publishManifest(s3, release);
+  process.exit();
+};
+
+main();
\ No newline at end of file