From 12ffc222bb91978dd323357f817bcb3b8ccb0880 Mon Sep 17 00:00:00 2001 From: Cheng Zhao Date: Fri, 27 Sep 2013 19:15:56 +0800 Subject: [PATCH 1/6] Uses GitHub Releases API to download atom-shell. --- tasks/update-atom-shell-task.coffee | 70 +++++++++++++++++++++++++++-- 1 file changed, 66 insertions(+), 4 deletions(-) diff --git a/tasks/update-atom-shell-task.coffee b/tasks/update-atom-shell-task.coffee index 8a4eda03e..006001e50 100644 --- a/tasks/update-atom-shell-task.coffee +++ b/tasks/update-atom-shell-task.coffee @@ -6,6 +6,46 @@ request = require 'request' module.exports = (grunt) -> {spawn, mkdir, rm, cp} = require('./task-helpers')(grunt) + accessToken = null + getTokenFromKeychain = -> + accessToken ?= process.env['ATOM_ACCESS_TOKEN'] + + callAtomShellReposApi = (path, callback) -> + options = + url: "https://api.github.com/repos/atom/atom-shell#{path}" + proxy: process.env.http_proxy || process.env.https_proxy + headers: + authorization: "token #{getTokenFromKeychain()}" + accept: 'application/vnd.github.manifold-preview' + request options, (error, response, body) -> + body = JSON.parse(body) if not error? + callback(error, response, body) + + findReleaseIdFromAtomShellVersion = (version, callback) -> + callAtomShellReposApi '/releases', (error, response, data) -> + if error? + grunt.log.error('Cannot get releases of atom-shell') + callback(error) + else + for release in data when release.tag_name is version + callback(null, release.id) + return + grunt.log.error("There is no #{version} release of atom-shell") + callback(false) + + getAtomShellDownloadUrl = (version, releaseId, callback) -> + callAtomShellReposApi "/releases/#{releaseId}/assets", (error, response, data) -> + if error? + grunt.log.error("Cannot get assets of atom-shell's #{version} release") + callback(error) + else + filename = "atom-shell-#{version}-#{process.platform}.zip" + for asset in data when asset.name is filename and asset.state is 'uploaded' + callback(null, asset.url) + return + grunt.log.error("Cannot get url of atom-shell's release asset") + callback(false) + getAtomShellVersion = -> versionPath = path.join('atom-shell', 'version') if grunt.file.isFile(versionPath) @@ -18,14 +58,25 @@ module.exports = (grunt) -> isAtomShellVersionCached = (version) -> grunt.file.isFile(getCachePath(version), 'version') - downloadAtomShell = (version, callback) -> + downloadAtomShell = (version, url, callback) -> options = - url: "https://gh-contractor-zcbenz.s3.amazonaws.com/atom-shell/#{version}/atom-shell-#{version}-darwin.zip" + url: url + followRedirect: false proxy: process.env.http_proxy || process.env.https_proxy + # Only set headers for GitHub host, the url could also be a S3 link and + # setting headers for it would make the request fail. + if require('url').parse(url).hostname is 'api.github.com' + options.headers = + authorization: "token #{getTokenFromKeychain()}" + accept: 'application/octet-stream' + inputStream = request(options) inputStream.on 'response', (response) -> - if response.statusCode is 200 + if response.statusCode is 302 + # Manually handle redirection so headers would not be sent for S3. + downloadAtomShell(version, response.headers.location, callback) + else if response.statusCode is 200 grunt.log.writeln("Downloading atom-shell version #{version.cyan}") cacheDirectory = getCachePath(version) rm(cacheDirectory) @@ -41,6 +92,17 @@ module.exports = (grunt) -> grunt.log.error("atom-shell #{version.cyan} request failed") callback(false) + downloadAtomShellOfVersion = (version, callback) -> + findReleaseIdFromAtomShellVersion version, (error, releaseId) -> + if error? + callback(error) + else + getAtomShellDownloadUrl version, releaseId, (error, url) -> + if error? + callback(error) + else + downloadAtomShell version, url, callback + unzipAtomShell = (zipPath, callback) -> grunt.log.writeln('Unzipping atom-shell') directoryPath = path.dirname(zipPath) @@ -74,7 +136,7 @@ module.exports = (grunt) -> installAtomShell(atomShellVersion) rebuildNativeModules(currentAtomShellVersion, done) else - downloadAtomShell atomShellVersion, (error, zipPath) -> + downloadAtomShellOfVersion atomShellVersion, (error, zipPath) -> if zipPath? unzipAtomShell zipPath, (error) -> if error? From 2870edaf0636f93a93623a169b0554b1d52188df Mon Sep 17 00:00:00 2001 From: probablycorey Date: Fri, 27 Sep 2013 10:47:02 -0700 Subject: [PATCH 2/6] Output error message if GitHub api fails --- tasks/update-atom-shell-task.coffee | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tasks/update-atom-shell-task.coffee b/tasks/update-atom-shell-task.coffee index 006001e50..052db87e0 100644 --- a/tasks/update-atom-shell-task.coffee +++ b/tasks/update-atom-shell-task.coffee @@ -18,7 +18,9 @@ module.exports = (grunt) -> authorization: "token #{getTokenFromKeychain()}" accept: 'application/vnd.github.manifold-preview' request options, (error, response, body) -> - body = JSON.parse(body) if not error? + if not error? + body = JSON.parse(body) + error = new Error(body.message) if response.statusCode != 200 callback(error, response, body) findReleaseIdFromAtomShellVersion = (version, callback) -> @@ -110,7 +112,7 @@ module.exports = (grunt) -> spawn {cmd: 'unzip', args: [zipPath, '-d', directoryPath]}, (error) -> rm(zipPath) callback(error) - + rebuildNativeModules = (previousVersion, callback) -> newVersion = getAtomShellVersion() if newVersion and newVersion isnt previousVersion @@ -137,10 +139,12 @@ module.exports = (grunt) -> rebuildNativeModules(currentAtomShellVersion, done) else downloadAtomShellOfVersion atomShellVersion, (error, zipPath) -> - if zipPath? + if error? + done(error) + else if zipPath? unzipAtomShell zipPath, (error) -> if error? - done(false) + done(error) else grunt.log.writeln("Installing atom-shell #{atomShellVersion.cyan}") installAtomShell(atomShellVersion) From 2f511ec412467a66148c761ebe7993251491e020 Mon Sep 17 00:00:00 2001 From: probablycorey Date: Fri, 27 Sep 2013 10:47:11 -0700 Subject: [PATCH 3/6] Give error more descriptive title --- tasks/update-atom-shell-task.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/update-atom-shell-task.coffee b/tasks/update-atom-shell-task.coffee index 052db87e0..80a621721 100644 --- a/tasks/update-atom-shell-task.coffee +++ b/tasks/update-atom-shell-task.coffee @@ -26,7 +26,7 @@ module.exports = (grunt) -> findReleaseIdFromAtomShellVersion = (version, callback) -> callAtomShellReposApi '/releases', (error, response, data) -> if error? - grunt.log.error('Cannot get releases of atom-shell') + grunt.log.error('GitHub API failed to access atom-shell releases') callback(error) else for release in data when release.tag_name is version From 6fb400926fc6026aa5320962709c19627cbfa868 Mon Sep 17 00:00:00 2001 From: probablycorey Date: Fri, 27 Sep 2013 11:41:37 -0700 Subject: [PATCH 4/6] Allow task to ask for GitHub API token --- tasks/update-atom-shell-task.coffee | 94 ++++++++++++++++++----------- 1 file changed, 59 insertions(+), 35 deletions(-) diff --git a/tasks/update-atom-shell-task.coffee b/tasks/update-atom-shell-task.coffee index 80a621721..eeffb0c10 100644 --- a/tasks/update-atom-shell-task.coffee +++ b/tasks/update-atom-shell-task.coffee @@ -7,21 +7,34 @@ module.exports = (grunt) -> {spawn, mkdir, rm, cp} = require('./task-helpers')(grunt) accessToken = null - getTokenFromKeychain = -> + getTokenFromKeychain = (callback) -> accessToken ?= process.env['ATOM_ACCESS_TOKEN'] + if accessToken + callback(null, accessToken) + return + + spawn {cmd: 'security', args: ['-q', 'find-generic-password', '-ws', 'GitHub API Token']}, (error, result, code) -> + accessToken = result unless error? + callback(error, accessToken) callAtomShellReposApi = (path, callback) -> - options = - url: "https://api.github.com/repos/atom/atom-shell#{path}" - proxy: process.env.http_proxy || process.env.https_proxy - headers: - authorization: "token #{getTokenFromKeychain()}" - accept: 'application/vnd.github.manifold-preview' - request options, (error, response, body) -> - if not error? - body = JSON.parse(body) - error = new Error(body.message) if response.statusCode != 200 - callback(error, response, body) + getTokenFromKeychain (error, accessToken) -> + if error + callback(error) + return + + options = + url: "https://api.github.com/repos/atom/atom-shell#{path}" + proxy: process.env.http_proxy || process.env.https_proxy + headers: + authorization: "token #{accessToken}" + accept: 'application/vnd.github.manifold-preview' + + request options, (error, response, body) -> + if not error? + body = JSON.parse(body) + error = new Error(body.message) if response.statusCode != 200 + callback(error, response, body) findReleaseIdFromAtomShellVersion = (version, callback) -> callAtomShellReposApi '/releases', (error, response, data) -> @@ -60,7 +73,7 @@ module.exports = (grunt) -> isAtomShellVersionCached = (version) -> grunt.file.isFile(getCachePath(version), 'version') - downloadAtomShell = (version, url, callback) -> + getDownloadOptions = (version, url, callback) -> options = url: url followRedirect: false @@ -69,30 +82,41 @@ module.exports = (grunt) -> # Only set headers for GitHub host, the url could also be a S3 link and # setting headers for it would make the request fail. if require('url').parse(url).hostname is 'api.github.com' - options.headers = - authorization: "token #{getTokenFromKeychain()}" - accept: 'application/octet-stream' + getTokenFromKeychain (error, accessToken) -> + options.headers = + authorization: "token #{accessToken}" + accept: 'application/octet-stream' - inputStream = request(options) - inputStream.on 'response', (response) -> - if response.statusCode is 302 - # Manually handle redirection so headers would not be sent for S3. - downloadAtomShell(version, response.headers.location, callback) - else if response.statusCode is 200 - grunt.log.writeln("Downloading atom-shell version #{version.cyan}") - cacheDirectory = getCachePath(version) - rm(cacheDirectory) - mkdir(cacheDirectory) - cacheFile = path.join(cacheDirectory, 'atom-shell.zip') - outputStream = fs.createWriteStream(cacheFile) - outputStream.on 'close', -> callback(null, cacheFile) - inputStream.pipe(outputStream) - else - if response.statusCode is 404 - grunt.log.error("atom-shell #{version.cyan} not found") + callback(error, options) + else + callback(null, options) + + downloadAtomShell = (version, url, callback) -> + getDownloadOptions version, url, (error, options) -> + if error + callback(error) + return + + inputStream = request(options) + inputStream.on 'response', (response) -> + if response.statusCode is 302 + # Manually handle redirection so headers would not be sent for S3. + downloadAtomShell(version, response.headers.location, callback) + else if response.statusCode is 200 + grunt.log.writeln("Downloading atom-shell version #{version.cyan}") + cacheDirectory = getCachePath(version) + rm(cacheDirectory) + mkdir(cacheDirectory) + cacheFile = path.join(cacheDirectory, 'atom-shell.zip') + outputStream = fs.createWriteStream(cacheFile) + outputStream.on 'close', -> callback(null, cacheFile) + inputStream.pipe(outputStream) else - grunt.log.error("atom-shell #{version.cyan} request failed") - callback(false) + if response.statusCode is 404 + grunt.log.error("atom-shell #{version.cyan} not found") + else + grunt.log.error("atom-shell #{version.cyan} request failed") + callback(false) downloadAtomShellOfVersion = (version, callback) -> findReleaseIdFromAtomShellVersion version, (error, releaseId) -> From d39797fdca8d9b6c56115ee7883b8ca004a11895 Mon Sep 17 00:00:00 2001 From: Cheng Zhao Date: Fri, 4 Oct 2013 12:42:15 +0800 Subject: [PATCH 5/6] Import atomcredentials in cibuild. --- script/cibuild | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/script/cibuild b/script/cibuild index 9f0dbdfc9..8742fe7c6 100755 --- a/script/cibuild +++ b/script/cibuild @@ -7,6 +7,12 @@ cd "$(dirname "$0")/.." rm -rf ~/.atom git clean -dff +ATOM_CREDENTIALS_FILE=/var/lib/jenkins/config/atomcredentials +if [ -f $ATOM_CREDENTIALS_FILE ]; then + . $ATOM_CREDENTIALS_FILE + export ATOM_ACCESS_TOKEN=$ATOM_ACCESS_TOKEN # make it visibile to grunt. +fi + ./script/bootstrap ./node_modules/.bin/apm clean ./node_modules/.bin/grunt ci --stack --no-color From 15a62eab63f7ec0fdcee9a9d47edb7071e0e49cd Mon Sep 17 00:00:00 2001 From: Matt Colyer Date: Fri, 4 Oct 2013 09:24:19 -0700 Subject: [PATCH 6/6] Parse multipart response from AWS S3 --- package.json | 1 + tasks/update-atom-shell-task.coffee | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index b787365c3..550014a21 100644 --- a/package.json +++ b/package.json @@ -117,6 +117,7 @@ }, "devDependencies": { "biscotto": "0.0.17", + "formidable": "~1.0.14", "fstream": "0.1.24", "grunt": "~0.4.1", "grunt-cli": "~0.1.9", diff --git a/tasks/update-atom-shell-task.coffee b/tasks/update-atom-shell-task.coffee index eeffb0c10..0e6460316 100644 --- a/tasks/update-atom-shell-task.coffee +++ b/tasks/update-atom-shell-task.coffee @@ -2,6 +2,7 @@ fs = require 'fs' path = require 'path' request = require 'request' +formidable = require 'formidable' module.exports = (grunt) -> {spawn, mkdir, rm, cp} = require('./task-helpers')(grunt) @@ -107,10 +108,17 @@ module.exports = (grunt) -> cacheDirectory = getCachePath(version) rm(cacheDirectory) mkdir(cacheDirectory) - cacheFile = path.join(cacheDirectory, 'atom-shell.zip') - outputStream = fs.createWriteStream(cacheFile) - outputStream.on 'close', -> callback(null, cacheFile) - inputStream.pipe(outputStream) + + form = new formidable.IncomingForm() + form.uploadDir = cacheDirectory + form.maxFieldsSize = 100 * 1024 * 1024 + form.on 'file', (name, file) -> + cacheFile = path.join(cacheDirectory, 'atom-shell.zip') + fs.renameSync(file.path, cacheFile) + callback(null, cacheFile) + form.parse response, (error) -> + if error + grunt.log.error("atom-shell #{version.cyan} failed to download") else if response.statusCode is 404 grunt.log.error("atom-shell #{version.cyan} not found")