diff --git a/package.json b/package.json index ccb50f64e..144bd0651 100644 --- a/package.json +++ b/package.json @@ -115,6 +115,7 @@ }, "devDependencies": { "biscotto": "0.0.17", + "formidable": "~1.0.14", "fstream": "0.1.24", "grunt": "~0.4.1", "grunt-cli": "~0.1.9", diff --git a/script/cibuild b/script/cibuild index 9f0dbdfc9..8742fe7c6 100755 --- a/script/cibuild +++ b/script/cibuild @@ -7,6 +7,12 @@ cd "$(dirname "$0")/.." rm -rf ~/.atom git clean -dff +ATOM_CREDENTIALS_FILE=/var/lib/jenkins/config/atomcredentials +if [ -f $ATOM_CREDENTIALS_FILE ]; then + . $ATOM_CREDENTIALS_FILE + export ATOM_ACCESS_TOKEN=$ATOM_ACCESS_TOKEN # make it visibile to grunt. +fi + ./script/bootstrap ./node_modules/.bin/apm clean ./node_modules/.bin/grunt ci --stack --no-color diff --git a/tasks/update-atom-shell-task.coffee b/tasks/update-atom-shell-task.coffee index 8a4eda03e..0e6460316 100644 --- a/tasks/update-atom-shell-task.coffee +++ b/tasks/update-atom-shell-task.coffee @@ -2,10 +2,66 @@ fs = require 'fs' path = require 'path' request = require 'request' +formidable = require 'formidable' module.exports = (grunt) -> {spawn, mkdir, rm, cp} = require('./task-helpers')(grunt) + accessToken = null + getTokenFromKeychain = (callback) -> + accessToken ?= process.env['ATOM_ACCESS_TOKEN'] + if accessToken + callback(null, accessToken) + return + + spawn {cmd: 'security', args: ['-q', 'find-generic-password', '-ws', 'GitHub API Token']}, (error, result, code) -> + accessToken = result unless error? + callback(error, accessToken) + + callAtomShellReposApi = (path, callback) -> + getTokenFromKeychain (error, accessToken) -> + if error + callback(error) + return + + options = + url: "https://api.github.com/repos/atom/atom-shell#{path}" + proxy: process.env.http_proxy || process.env.https_proxy + headers: + authorization: "token #{accessToken}" + accept: 'application/vnd.github.manifold-preview' + + request options, (error, response, body) -> + if not error? + body = JSON.parse(body) + error = new Error(body.message) if response.statusCode != 200 + callback(error, response, body) + + findReleaseIdFromAtomShellVersion = (version, callback) -> + callAtomShellReposApi '/releases', (error, response, data) -> + if error? + grunt.log.error('GitHub API failed to access atom-shell releases') + callback(error) + else + for release in data when release.tag_name is version + callback(null, release.id) + return + grunt.log.error("There is no #{version} release of atom-shell") + callback(false) + + getAtomShellDownloadUrl = (version, releaseId, callback) -> + callAtomShellReposApi "/releases/#{releaseId}/assets", (error, response, data) -> + if error? + grunt.log.error("Cannot get assets of atom-shell's #{version} release") + callback(error) + else + filename = "atom-shell-#{version}-#{process.platform}.zip" + for asset in data when asset.name is filename and asset.state is 'uploaded' + callback(null, asset.url) + return + grunt.log.error("Cannot get url of atom-shell's release asset") + callback(false) + getAtomShellVersion = -> versionPath = path.join('atom-shell', 'version') if grunt.file.isFile(versionPath) @@ -18,28 +74,68 @@ module.exports = (grunt) -> isAtomShellVersionCached = (version) -> grunt.file.isFile(getCachePath(version), 'version') - downloadAtomShell = (version, callback) -> + getDownloadOptions = (version, url, callback) -> options = - url: "https://gh-contractor-zcbenz.s3.amazonaws.com/atom-shell/#{version}/atom-shell-#{version}-darwin.zip" + url: url + followRedirect: false proxy: process.env.http_proxy || process.env.https_proxy - inputStream = request(options) - inputStream.on 'response', (response) -> - if response.statusCode is 200 - grunt.log.writeln("Downloading atom-shell version #{version.cyan}") - cacheDirectory = getCachePath(version) - rm(cacheDirectory) - mkdir(cacheDirectory) - cacheFile = path.join(cacheDirectory, 'atom-shell.zip') - outputStream = fs.createWriteStream(cacheFile) - outputStream.on 'close', -> callback(null, cacheFile) - inputStream.pipe(outputStream) - else - if response.statusCode is 404 - grunt.log.error("atom-shell #{version.cyan} not found") + # Only set headers for GitHub host, the url could also be a S3 link and + # setting headers for it would make the request fail. + if require('url').parse(url).hostname is 'api.github.com' + getTokenFromKeychain (error, accessToken) -> + options.headers = + authorization: "token #{accessToken}" + accept: 'application/octet-stream' + + callback(error, options) + else + callback(null, options) + + downloadAtomShell = (version, url, callback) -> + getDownloadOptions version, url, (error, options) -> + if error + callback(error) + return + + inputStream = request(options) + inputStream.on 'response', (response) -> + if response.statusCode is 302 + # Manually handle redirection so headers would not be sent for S3. + downloadAtomShell(version, response.headers.location, callback) + else if response.statusCode is 200 + grunt.log.writeln("Downloading atom-shell version #{version.cyan}") + cacheDirectory = getCachePath(version) + rm(cacheDirectory) + mkdir(cacheDirectory) + + form = new formidable.IncomingForm() + form.uploadDir = cacheDirectory + form.maxFieldsSize = 100 * 1024 * 1024 + form.on 'file', (name, file) -> + cacheFile = path.join(cacheDirectory, 'atom-shell.zip') + fs.renameSync(file.path, cacheFile) + callback(null, cacheFile) + form.parse response, (error) -> + if error + grunt.log.error("atom-shell #{version.cyan} failed to download") else - grunt.log.error("atom-shell #{version.cyan} request failed") - callback(false) + if response.statusCode is 404 + grunt.log.error("atom-shell #{version.cyan} not found") + else + grunt.log.error("atom-shell #{version.cyan} request failed") + callback(false) + + downloadAtomShellOfVersion = (version, callback) -> + findReleaseIdFromAtomShellVersion version, (error, releaseId) -> + if error? + callback(error) + else + getAtomShellDownloadUrl version, releaseId, (error, url) -> + if error? + callback(error) + else + downloadAtomShell version, url, callback unzipAtomShell = (zipPath, callback) -> grunt.log.writeln('Unzipping atom-shell') @@ -48,7 +144,7 @@ module.exports = (grunt) -> spawn {cmd: 'unzip', args: [zipPath, '-d', directoryPath]}, (error) -> rm(zipPath) callback(error) - + rebuildNativeModules = (previousVersion, callback) -> newVersion = getAtomShellVersion() if newVersion and newVersion isnt previousVersion @@ -74,11 +170,13 @@ module.exports = (grunt) -> installAtomShell(atomShellVersion) rebuildNativeModules(currentAtomShellVersion, done) else - downloadAtomShell atomShellVersion, (error, zipPath) -> - if zipPath? + downloadAtomShellOfVersion atomShellVersion, (error, zipPath) -> + if error? + done(error) + else if zipPath? unzipAtomShell zipPath, (error) -> if error? - done(false) + done(error) else grunt.log.writeln("Installing atom-shell #{atomShellVersion.cyan}") installAtomShell(atomShellVersion)