Merge pull request #890 from atom/atom-shell-with-releases-api

Uses GitHub Releases API to download atom-shell.

package.json

@@ -115,6 +115,7 @@
   },
   "devDependencies": {
     "biscotto": "0.0.17",
+    "formidable": "~1.0.14",
     "fstream": "0.1.24",
     "grunt": "~0.4.1",
     "grunt-cli": "~0.1.9",

script/cibuild

@@ -7,6 +7,12 @@ cd "$(dirname "$0")/.."
 rm -rf ~/.atom
 git clean -dff
 
+ATOM_CREDENTIALS_FILE=/var/lib/jenkins/config/atomcredentials
+if [ -f $ATOM_CREDENTIALS_FILE ]; then
+  . $ATOM_CREDENTIALS_FILE
+  export ATOM_ACCESS_TOKEN=$ATOM_ACCESS_TOKEN # make it visibile to grunt.
+fi
+
 ./script/bootstrap
 ./node_modules/.bin/apm clean
 ./node_modules/.bin/grunt ci --stack --no-color

tasks/update-atom-shell-task.coffee

@@ -2,10 +2,66 @@ fs = require 'fs'
 path = require 'path'
 
 request = require 'request'
+formidable = require 'formidable'
 
 module.exports = (grunt) ->
   {spawn, mkdir, rm, cp} = require('./task-helpers')(grunt)
 
+  accessToken = null
+  getTokenFromKeychain = (callback) ->
+    accessToken ?= process.env['ATOM_ACCESS_TOKEN']
+    if accessToken
+      callback(null, accessToken)
+      return
+
+    spawn {cmd: 'security', args: ['-q', 'find-generic-password', '-ws', 'GitHub API Token']}, (error, result, code) ->
+      accessToken = result unless error?
+      callback(error, accessToken)
+
+  callAtomShellReposApi = (path, callback) ->
+    getTokenFromKeychain (error, accessToken) ->
+      if error
+        callback(error)
+        return
+
+      options =
+        url: "https://api.github.com/repos/atom/atom-shell#{path}"
+        proxy: process.env.http_proxy || process.env.https_proxy
+        headers:
+          authorization: "token #{accessToken}"
+          accept: 'application/vnd.github.manifold-preview'
+
+      request options, (error, response, body) ->
+        if not error?
+          body = JSON.parse(body)
+          error = new Error(body.message) if response.statusCode != 200
+        callback(error, response, body)
+
+  findReleaseIdFromAtomShellVersion = (version, callback) ->
+    callAtomShellReposApi '/releases', (error, response, data) ->
+      if error?
+        grunt.log.error('GitHub API failed to access atom-shell releases')
+        callback(error)
+      else
+        for release in data when release.tag_name is version
+          callback(null, release.id)
+          return
+        grunt.log.error("There is no #{version} release of atom-shell")
+        callback(false)
+
+  getAtomShellDownloadUrl = (version, releaseId, callback) ->
+    callAtomShellReposApi "/releases/#{releaseId}/assets", (error, response, data) ->
+      if error?
+        grunt.log.error("Cannot get assets of atom-shell's #{version} release")
+        callback(error)
+      else
+        filename = "atom-shell-#{version}-#{process.platform}.zip"
+        for asset in data when asset.name is filename and asset.state is 'uploaded'
+          callback(null, asset.url)
+          return
+        grunt.log.error("Cannot get url of atom-shell's release asset")
+        callback(false)
+
   getAtomShellVersion = ->
     versionPath = path.join('atom-shell', 'version')
     if grunt.file.isFile(versionPath)
@@ -18,28 +74,68 @@ module.exports = (grunt) ->
   isAtomShellVersionCached = (version) ->
     grunt.file.isFile(getCachePath(version), 'version')
 
-  downloadAtomShell = (version, callback) ->
+  getDownloadOptions = (version, url, callback) ->
     options =
-      url: "https://gh-contractor-zcbenz.s3.amazonaws.com/atom-shell/#{version}/atom-shell-#{version}-darwin.zip"
+      url: url
+      followRedirect: false
       proxy: process.env.http_proxy || process.env.https_proxy
 
-    inputStream = request(options)
-    inputStream.on 'response', (response) ->
-      if response.statusCode is 200
-        grunt.log.writeln("Downloading atom-shell version #{version.cyan}")
-        cacheDirectory = getCachePath(version)
-        rm(cacheDirectory)
-        mkdir(cacheDirectory)
-        cacheFile = path.join(cacheDirectory, 'atom-shell.zip')
-        outputStream = fs.createWriteStream(cacheFile)
-        outputStream.on 'close', -> callback(null, cacheFile)
-        inputStream.pipe(outputStream)
-      else
-        if response.statusCode is 404
-          grunt.log.error("atom-shell #{version.cyan} not found")
+    # Only set headers for GitHub host, the url could also be a S3 link and
+    # setting headers for it would make the request fail.
+    if require('url').parse(url).hostname is 'api.github.com'
+      getTokenFromKeychain (error, accessToken) ->
+        options.headers =
+          authorization: "token #{accessToken}"
+          accept: 'application/octet-stream'
+
+        callback(error, options)
+    else
+      callback(null, options)
+
+  downloadAtomShell = (version, url, callback) ->
+    getDownloadOptions version, url, (error, options) ->
+      if error
+        callback(error)
+        return
+
+      inputStream = request(options)
+      inputStream.on 'response', (response) ->
+        if response.statusCode is 302
+          # Manually handle redirection so headers would not be sent for S3.
+          downloadAtomShell(version, response.headers.location, callback)
+        else if response.statusCode is 200
+          grunt.log.writeln("Downloading atom-shell version #{version.cyan}")
+          cacheDirectory = getCachePath(version)
+          rm(cacheDirectory)
+          mkdir(cacheDirectory)
+
+          form = new formidable.IncomingForm()
+          form.uploadDir = cacheDirectory
+          form.maxFieldsSize = 100 * 1024 * 1024
+          form.on 'file', (name, file) ->
+            cacheFile = path.join(cacheDirectory, 'atom-shell.zip')
+            fs.renameSync(file.path, cacheFile)
+            callback(null, cacheFile)
+          form.parse response, (error) ->
+            if error
+              grunt.log.error("atom-shell #{version.cyan} failed to download")
         else
-          grunt.log.error("atom-shell #{version.cyan} request failed")
-        callback(false)
+          if response.statusCode is 404
+            grunt.log.error("atom-shell #{version.cyan} not found")
+          else
+            grunt.log.error("atom-shell #{version.cyan} request failed")
+          callback(false)
+
+  downloadAtomShellOfVersion = (version, callback) ->
+    findReleaseIdFromAtomShellVersion version, (error, releaseId) ->
+      if error?
+        callback(error)
+      else
+        getAtomShellDownloadUrl version, releaseId, (error, url) ->
+          if error?
+            callback(error)
+          else
+            downloadAtomShell version, url, callback
 
   unzipAtomShell = (zipPath, callback) ->
     grunt.log.writeln('Unzipping atom-shell')
@@ -48,7 +144,7 @@ module.exports = (grunt) ->
     spawn {cmd: 'unzip', args: [zipPath, '-d', directoryPath]}, (error) ->
       rm(zipPath)
       callback(error)
-        
+
   rebuildNativeModules = (previousVersion, callback) ->
     newVersion = getAtomShellVersion()
     if newVersion and newVersion isnt previousVersion
@@ -74,11 +170,13 @@ module.exports = (grunt) ->
           installAtomShell(atomShellVersion)
           rebuildNativeModules(currentAtomShellVersion, done)
         else
-          downloadAtomShell atomShellVersion, (error, zipPath) ->
-            if zipPath?
+          downloadAtomShellOfVersion atomShellVersion, (error, zipPath) ->
+            if error?
+              done(error)
+            else if zipPath?
               unzipAtomShell zipPath, (error) ->
                 if error?
-                  done(false)
+                  done(error)
                 else
                   grunt.log.writeln("Installing atom-shell #{atomShellVersion.cyan}")
                   installAtomShell(atomShellVersion)