From 5b07154b8e4b6ec5bcbd36093d6ca891808026a8 Mon Sep 17 00:00:00 2001 From: Mark Lee Date: Sun, 24 Jul 2016 19:19:23 -0700 Subject: [PATCH 1/2] Generate and upload checksums for released ZIPs to GitHub When generating an Electron release, create a `sha256sum`-compatible file for each ZIP file, and upload them to the corresponding GitHub release. This is primarily to confirm that the download of a given ZIP completed successfully, as opposed to verifying that an Electron team member uploaded the given ZIP files (which would require using a trusted GPG key). --- script/lib/util.py | 14 ++++++++++++++ script/upload.py | 21 +++++++++++++++------ 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/script/lib/util.py b/script/lib/util.py index 4db4e5fa67..6833af99a6 100644 --- a/script/lib/util.py +++ b/script/lib/util.py @@ -3,6 +3,7 @@ import atexit import contextlib import errno +import hashlib import platform import re import shutil @@ -129,6 +130,19 @@ def make_zip(zip_file_path, files, dirs): for f in filenames: zip_file.write(os.path.join(root, f)) zip_file.close() + make_zip_sha256_checksum(zip_file_path) + + +def make_zip_sha256_checksum(zip_file_path): + checksum_path = '{}.sha256sum'.format(zip_file_path) + safe_unlink(checksum_path) + sha256 = hashlib.sha256() + with open(zip_file_path, 'rb') as f: + sha256.update(f.read()) + + zip_basename = os.path.basename(zip_file_path) + with open(checksum_path, 'w') as checksum: + checksum.write('{} *{}'.format(sha256.hexdigest(), zip_basename)) def rm_rf(path): diff --git a/script/upload.py b/script/upload.py index 1abd67aaf0..e7883e322e 100755 --- a/script/upload.py +++ b/script/upload.py @@ -203,20 +203,29 @@ def create_release_draft(github, tag): def upload_electron(github, release, file_path): - # Delete the original file before uploading in CI. + checksum_path = '{}.sha256sum'.format(file_path) + # Delete the original file & its checksum before uploading in CI. + filename = os.path.basename(file_path) + checksum_filename = os.path.basename(checksum_path) if os.environ.has_key('CI'): try: for asset in release['assets']: - if asset['name'] == os.path.basename(file_path): + if asset['name'] in [filename, checksum_filename]: github.repos(ELECTRON_REPO).releases.assets(asset['id']).delete() - break except Exception: pass # Upload the file. - params = {'name': os.path.basename(file_path)} - headers = {'Content-Type': 'application/zip'} - with open(file_path, 'rb') as f: + upload_asset_to_github(github, release, file_path, 'application/zip') + + # Upload the file's checksum. + upload_asset_to_github(github, release, checksum_path, 'text/plain') + + +def upload_asset_to_github(github, release, asset_path, content_type): + params = {'name': os.path.dirname(asset_path)} + headers = {'Content-Type': content_type} + with open(asset_path) as f: github.repos(ELECTRON_REPO).releases(release['id']).assets.post( params=params, headers=headers, data=f, verify=False) From 59de146a9fea8b183f030b07eddb9c5c8ddd4f4a Mon Sep 17 00:00:00 2001 From: Mark Lee Date: Sun, 31 Jul 2016 19:00:19 -0700 Subject: [PATCH 2/2] Use S3 as an intermediary store & merge shasum files before uploading to GitHub --- script/lib/util.py | 47 ++++++++++----- script/merge-electron-checksums.py | 41 +++++++++++++ ...-checksums.py => upload-node-checksums.py} | 0 script/upload.py | 57 ++++++++++--------- 4 files changed, 104 insertions(+), 41 deletions(-) create mode 100644 script/merge-electron-checksums.py rename script/{upload-checksums.py => upload-node-checksums.py} (100%) diff --git a/script/lib/util.py b/script/lib/util.py index 6833af99a6..933144d95c 100644 --- a/script/lib/util.py +++ b/script/lib/util.py @@ -16,9 +16,12 @@ import urllib2 import os import zipfile -from config import is_verbose_mode +from config import is_verbose_mode, s3_config from env_util import get_vs_env +BOTO_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..', 'vendor', + 'boto')) + def get_host_arch(): """Returns the host architecture with a predictable string.""" @@ -130,10 +133,11 @@ def make_zip(zip_file_path, files, dirs): for f in filenames: zip_file.write(os.path.join(root, f)) zip_file.close() - make_zip_sha256_checksum(zip_file_path) + upload_zip_sha256_checksum(zip_file_path) -def make_zip_sha256_checksum(zip_file_path): +def upload_zip_sha256_checksum(zip_file_path): + bucket, access_key, secret_key = s3_config() checksum_path = '{}.sha256sum'.format(zip_file_path) safe_unlink(checksum_path) sha256 = hashlib.sha256() @@ -143,6 +147,8 @@ def make_zip_sha256_checksum(zip_file_path): zip_basename = os.path.basename(zip_file_path) with open(checksum_path, 'w') as checksum: checksum.write('{} *{}'.format(sha256.hexdigest(), zip_basename)) + s3put(bucket, access_key, secret_key, os.path.dirname(checksum_path), + 'atom-shell/tmp', [checksum_path]) def rm_rf(path): @@ -216,28 +222,39 @@ def parse_version(version): return vs + ['0'] * (4 - len(vs)) -def s3put(bucket, access_key, secret_key, prefix, key_prefix, files): - env = os.environ.copy() - BOTO_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..', 'vendor', - 'boto')) - env['PYTHONPATH'] = os.path.pathsep.join([ - env.get('PYTHONPATH', ''), +def boto_path_dirs(): + return [ os.path.join(BOTO_DIR, 'build', 'lib'), - os.path.join(BOTO_DIR, 'build', 'lib.linux-x86_64-2.7')]) + os.path.join(BOTO_DIR, 'build', 'lib.linux-x86_64-2.7') + ] + + +def run_boto_script(access_key, secret_key, script_name, *args): + env = os.environ.copy() + env['AWS_ACCESS_KEY_ID'] = access_key + env['AWS_SECRET_ACCESS_KEY'] = secret_key + env['PYTHONPATH'] = os.path.pathsep.join( + [env.get('PYTHONPATH', '')] + boto_path_dirs()) + + boto = os.path.join(BOTO_DIR, 'bin', script_name) - boto = os.path.join(BOTO_DIR, 'bin', 's3put') args = [ sys.executable, - boto, + boto + ] + args + + execute(args, env) + + +def s3put(bucket, access_key, secret_key, prefix, key_prefix, files): + args = [ '--bucket', bucket, - '--access_key', access_key, - '--secret_key', secret_key, '--prefix', prefix, '--key_prefix', key_prefix, '--grant', 'public-read' ] + files - execute(args, env) + run_boto_script(access_key, secret_key, 's3put', *args) def import_vs_env(target_arch): diff --git a/script/merge-electron-checksums.py b/script/merge-electron-checksums.py new file mode 100644 index 0000000000..7a5be5fe14 --- /dev/null +++ b/script/merge-electron-checksums.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python + +# Download individual checksum files for Electron zip files from S3, +# concatenate them, and upload to GitHub. + +from __future__ import print_function + +import argparse +import sys + +from lib.config import s3_config +from lib.util import boto_path_dirs + +sys.path.extend(boto_path_dirs()) + +from boto.s3.connection import S3Connection + +def main(): + args = parse_args() + bucket_name, access_key, secret_key = s3_config() + s3 = S3Connection(access_key, secret_key) + bucket = s3.get_bucket(bucket_name) + if bucket is None: + print('S3 bucket "{}" does not exist!'.format(bucket_name), file=sys.stderr) + return 1 + shasums = [s3_object.get_contents_as_string().strip() + for s3_object in bucket.list('atom-shell/tmp/', delimiter='/') + if s3_object.key.endswith('.sha256sum') and + args.version in s3_object.key] + print('\n'.join(shasums)) + return 0 + + +def parse_args(): + parser = argparse.ArgumentParser(description='Upload SHASUMS files to GitHub') + parser.add_argument('-v', '--version', help='Specify the version', + required=True) + return parser.parse_args() + +if __name__ == '__main__': + sys.exit(main()) diff --git a/script/upload-checksums.py b/script/upload-node-checksums.py similarity index 100% rename from script/upload-checksums.py rename to script/upload-node-checksums.py diff --git a/script/upload.py b/script/upload.py index e7883e322e..e41946dffe 100755 --- a/script/upload.py +++ b/script/upload.py @@ -2,6 +2,7 @@ import argparse import errno +from io import StringIO import os import subprocess import sys @@ -46,8 +47,7 @@ def main(): if not args.publish_release: if not dist_newer_than_head(): - create_dist = os.path.join(SOURCE_ROOT, 'script', 'create-dist.py') - execute([sys.executable, create_dist]) + run_python_script('create-dist.py') build_version = get_electron_build_version() if not ELECTRON_VERSION.startswith(build_version): @@ -69,14 +69,14 @@ def main(): tag_exists) if args.publish_release: - # Upload the SHASUMS.txt. - execute([sys.executable, - os.path.join(SOURCE_ROOT, 'script', 'upload-checksums.py'), - '-v', ELECTRON_VERSION]) + # Upload the Node SHASUMS*.txt. + run_python_script('upload-node-checksums.py', '-v', ELECTRON_VERSION) # Upload the index.json. - execute([sys.executable, - os.path.join(SOURCE_ROOT, 'script', 'upload-index-json.py')]) + run_python_script('upload-index-json.py') + + # Create and upload the Electron SHASUMS*.txt + release_electron_checksums(github, release) # Press the publish button. publish_release(github, release['id']) @@ -108,13 +108,10 @@ def main(): if PLATFORM == 'win32' and not tag_exists: # Upload PDBs to Windows symbol server. - execute([sys.executable, - os.path.join(SOURCE_ROOT, 'script', 'upload-windows-pdb.py')]) + run_python_script('upload-windows-pdb.py') # Upload node headers. - execute([sys.executable, - os.path.join(SOURCE_ROOT, 'script', 'upload-node-headers.py'), - '-v', args.version]) + run_python_script('upload-node-headers.py', '-v', args.version) def parse_args(): @@ -127,6 +124,11 @@ def parse_args(): return parser.parse_args() +def run_python_script(script, *args): + script_path = os.path.join(SOURCE_ROOT, 'script', script), + return execute([sys.executable, script_path] + args) + + def get_electron_build_version(): if get_target_arch() == 'arm' or os.environ.has_key('CI'): # In CI we just build as told. @@ -202,32 +204,35 @@ def create_release_draft(github, tag): return r +def release_electron_checksums(github, release): + checksums = run_python_script( + 'merge-electron-checksums.py', '-v', ELECTRON_VERSION) + upload_io_to_github(github, release, + 'SHASUMS256.txt', StringIO(checksums), 'text/plain') + + def upload_electron(github, release, file_path): - checksum_path = '{}.sha256sum'.format(file_path) - # Delete the original file & its checksum before uploading in CI. + # Delete the original file before uploading in CI. filename = os.path.basename(file_path) - checksum_filename = os.path.basename(checksum_path) if os.environ.has_key('CI'): try: for asset in release['assets']: - if asset['name'] in [filename, checksum_filename]: + if asset['name'] == filename: github.repos(ELECTRON_REPO).releases.assets(asset['id']).delete() except Exception: pass # Upload the file. - upload_asset_to_github(github, release, file_path, 'application/zip') - - # Upload the file's checksum. - upload_asset_to_github(github, release, checksum_path, 'text/plain') + name = os.path.dirname(file_path) + with open(file_path, 'rb') as f: + upload_io_to_github(github, release, name, f, 'application/zip') -def upload_asset_to_github(github, release, asset_path, content_type): - params = {'name': os.path.dirname(asset_path)} +def upload_io_to_github(github, release, name, io, content_type): + params = {'name': name} headers = {'Content-Type': content_type} - with open(asset_path) as f: - github.repos(ELECTRON_REPO).releases(release['id']).assets.post( - params=params, headers=headers, data=f, verify=False) + github.repos(ELECTRON_REPO).releases(release['id']).assets.post( + params=params, headers=headers, data=io, verify=False) def publish_release(github, release_id):