Cleaning up the build processes

This commit is contained in:
Jorrit Wronski
2015-07-09 18:43:33 +02:00
parent ae7b771559
commit 91398950cb
3 changed files with 129 additions and 67 deletions

View File

@@ -237,37 +237,87 @@ class PythonSlaveConfig(object):
return IDs
@properties.renderer
def gitModeInput(props):
""" If we are doing a full clean, this will tell it to clobber all the files """
if props.getProperty('fullclean', default = False) or props.getProperty('branch') == 'release':
return 'full'
else:
return 'incremental'
import os
@properties.renderer
def masterdestLocation(props):
def _branch(props): return props.getProperty('branch')
def _master_loc_rel(git_branch):
"""
If building from release branch, upload to public_html/release
If building from master branch, upload to normal public_html/binaries folder
If another branch, upload to public_html/unstable
"""
if props.getProperty('branch') == 'release':
return 'public_html/release'
elif props.getProperty('branch') == 'master':
return 'public_html/binaries'
else:
return 'public_html/unstable'
if git_branch == 'release': return os.path.join('public_html','release')
elif git_branch == 'master': return os.path.join('public_html','binaries')
else: return os.path.join('public_html','unstable')
@properties.renderer
def master_loc_rel(props):
"""
If building from release branch, upload to public_html/release
If building from master branch, upload to normal public_html/binaries folder
If another branch, upload to public_html/unstable
"""
return _master_loc_rel(props.getProperty('branch'))
def _master_loc_abs(git_branch):
server_uri = 'coolprop@coolprop.dreamhosters.com'
server_dir = '/home/coolprop/buildbot/server-master'
server_dir = os.path.join(server_dir, _master_loc_rel(git_branch))
server_des = "{0}:{1}".format(server_uri, os.path.abspath(server_dir))
return server_des
@properties.renderer
def master_loc_abs(props):
return _master_loc_abs(props.getProperty('branch'))
############# Upload folder permissions #######################
def fixPermissions(factory):
factory.addStep(MasterShellCommand(command = '${HOME}/scripts/binPerms.sh'))
def upload_command(factory, slavesrc, masterdest=None, branch=None, platform=None, pyID=None):
"""Upload files to the master server. Avoids buildbot upload on platforms other than Windows."""
if (masterdest is None and branch is None) or \
(masterdest is not None and branch is not None):
raise ValueError("Unknown target, specify either \"masterdest\" or \"branch\".")
if (platform is not None and pyID is not None):
raise ValueError("Unknown target, specify either \"platform\" or \"pyID\".")
if platform is None and pyID is None: # default
if masterdest is not None: target = masterdest
else: target = _master_loc_rel(branch)
factory.addStep(DirectoryUpload(slavesrc=slavesrc, masterdest=target))
elif (platform is not None and platform == 'windows') or \
(pyID is not None and checkID(pyID, teID=100, strict=False)):
if masterdest is not None: target = masterdest
else: target = _master_loc_rel(branch)
factory.addStep(DirectoryUpload(slavesrc=slavesrc, masterdest=target))
else:
if masterdest is not None: target = masterdest
else: target = _master_loc_abs(branch)
rsyncCommand = ' '.join(['rsync', '-a', '--stats', '{0}/ {1}'.format(slavesrc,masterdest)])
factory.addStep(ShellCommand(command=rsyncCommand, haltOnFailure = False))
fixPermissions(factory)
# A centralised method to provide the objects with some presets
def getBaseFactory(gitMode = 'incremental'):
@properties.renderer
def _git_mode(props):
""" If we are doing a full clean, this will tell it to clobber all the files """
if props.getProperty('fullclean',default=False) or props.getProperty('branch',default='master') == 'release':
return 'full'
else:
return 'incremental'
def getBaseFactory():
factory = BuildFactory()
factory.addStep(Git(repourl='git://github.com/CoolProp/CoolProp', mode=gitModeInput, method = 'fresh', submodules = True, progress=True, haltOnFailure = True))
factory.addStep(Git(
repourl= 'git://github.com/CoolProp/CoolProp',
mode = _git_mode,
method = 'fresh',
submodules = True,
progress=True,
haltOnFailure = True))
return factory
def docActivateCmd():
@@ -308,7 +358,7 @@ def websiteFactory(platform, gitMode='incremental', fullBuild=False):
raise ValueError("The docs cannot be build on a Windows machine, we rely on rsync...")
#
# Create the factory to add the actions to
factory = getBaseFactory(gitMode=gitMode)
factory = getBaseFactory()
# Make a wheel - this is advantageous because it forces pip to uninstall coolprop, ensuring that all files installed are from this wheel
factory.addStep(ShellCommand(command=' '.join([docActivateCmd(), "python", "setup.py", "bdist_wheel", '--dist-dir', 'dist']), workdir= 'build/wrappers/Python', haltOnFailure = True))
# List the files in the dist directory
@@ -349,7 +399,7 @@ def pythonFactory(pyID, pyCFG=PythonSlaveConfig("name"), gitMode='incremental'):
relinstFolder = "../../install_root" # relative path
uploadFolder = "install_root" # relative to build
# Create the factory to add the actions to
factory = getBaseFactory(gitMode=gitMode)
factory = getBaseFactory()
# Remove the temporary folder for installs
factory.addStep(RemoveDirectory(dir=installFolder, haltOnFailure = False))
#
@@ -378,17 +428,17 @@ def pythonFactory(pyID, pyCFG=PythonSlaveConfig("name"), gitMode='incremental'):
#factory.addStep(ShellCommand(command="python -c \"import CoolProp; print(CoolProp.__file___)\"", workdir= workingFolder, haltOnFailure = True))
#
if not checkID(pyID, teID=300, strict=False): # Only upload non-Linux files
factory.addStep(DirectoryUpload(slavesrc=uploadFolder, masterdest=masterdestLocation, url="binaries", compress="bz2"))
factory.addStep(DirectoryUpload(slavesrc=uploadFolder, masterdest=master_loc_rel, url="binaries", compress="bz2"))
fixPermissions(factory)
#
pkgs = ["requests", "jinja2", "pyyaml", "numpy", "scipy", "matplotlib", "pandas", "cython"]
if checkID(pyID, teID=100, strict=False):
pkgs.extend(["unxutils","pywin32"]) # Add Windows-only dependency
if checkID(pyID, teID=101, strict=False):
pkgs.append("ndg-httpsclient") # Add Windows-only and python2 dependency
installCMD = " ".join(["conda", "install", "-yq"]) + " " + " ".join(pkgs)
combinedCMD = " && ".join([activateCMD, installCMD])
factory.addStep(ShellCommand(command = combinedCMD, workdir = workingFolder, haltOnFailure = False))
#pkgs = ["requests", "jinja2", "pyyaml", "numpy", "scipy", "matplotlib", "pandas", "cython"]
#if checkID(pyID, teID=100, strict=False):
# pkgs.extend(["unxutils","pywin32"]) # Add Windows-only dependency
# if checkID(pyID, teID=101, strict=False):
# pkgs.append("ndg-httpsclient") # Add Windows-only and python2 dependency
#installCMD = " ".join(["conda", "install", "-yq"]) + " " + " ".join(pkgs)
#combinedCMD = " && ".join([activateCMD, installCMD])
#factory.addStep(ShellCommand(command = combinedCMD, workdir = workingFolder, haltOnFailure = False))
#
installCMD = " ".join(["python", "generate_meta.yaml.py"])
combinedCMD = " && ".join([activateCMD, installCMD])
@@ -398,8 +448,9 @@ def pythonFactory(pyID, pyCFG=PythonSlaveConfig("name"), gitMode='incremental'):
combinedCMD = " && ".join([activateCMD, installCMD])
factory.addStep(ShellCommand(command = combinedCMD, workdir = workingFolder+"/../..", haltOnFailure = False))
#
factory.addStep(DirectoryUpload(slavesrc='conda', masterdest=masterdestLocation, compress="bz2"))
fixPermissions(factory)
#factory.addStep(DirectoryUpload(slavesrc='conda', masterdest=master_loc_rel, compress="bz2"))
#fixPermissions(factory)
upload_command(factory, 'conda', branch=_branch, pyID=pyID)
#
#installCMD = " ".join(["binstar", "upload", "--channel", "dev"])
#
@@ -430,7 +481,7 @@ def cmakeFactory(mod_name = None, gitMode = 'incremental', install = True, pre_c
working_folder = "build/build"
# Create the factory to add the actions to
factory = getBaseFactory(gitMode=gitMode)
factory = getBaseFactory()
#
factory.addStep(MakeDirectory(dir=working_folder, haltOnFailure = True))
factory.addStep(RemoveDirectory(dir="build/install_root", haltOnFailure = False))
@@ -462,7 +513,7 @@ def cmakeFactory(mod_name = None, gitMode = 'incremental', install = True, pre_c
if install:
factory.addStep(DirectoryUpload(
slavesrc="install_root",
masterdest=masterdestLocation,
masterdest=master_loc_rel,
url="binaries",
compress="bz2"))
return factory
@@ -511,7 +562,7 @@ def javascript_slave(platform, cmake_args = [], cmake_env = {}, build_args = [],
from buildbot.process.properties import WithProperties
# Create the factory to add the actions to
factory = getBaseFactory(gitMode=gitMode)
factory = getBaseFactory()
# Remove the temporary folder for installs
factory.addStep(RemoveDirectory(dir="build/install_root", haltOnFailure = False))
@@ -520,7 +571,7 @@ def javascript_slave(platform, cmake_args = [], cmake_env = {}, build_args = [],
workdir= working_folder,
haltOnFailure = True))
factory.addStep(ShellCommand(command=["cmake", "--build", ".", "--target", "install"]+build_args, workdir = working_folder, haltOnFailure = True))
factory.addStep(DirectoryUpload(slavesrc="install_root", masterdest=masterdestLocation, url="binaries", compress="bz2"))
factory.addStep(DirectoryUpload(slavesrc="install_root", masterdest=master_loc_rel, url="binaries", compress="bz2"))
fixPermissions(factory)
return factory
@@ -528,7 +579,7 @@ def python_source_slave(key, platform, conda_env, cmake_args = [], cmake_env = {
working_folder = "build/build"
# Create the factory to add the actions to
factory = getBaseFactory(gitMode=gitMode)
factory = getBaseFactory()
# Remove the temporary folder for installs
factory.addStep(RemoveDirectory(dir="build/install_root", haltOnFailure = False))
@@ -538,7 +589,7 @@ def python_source_slave(key, platform, conda_env, cmake_args = [], cmake_env = {
env = cmake_env,
workdir= working_folder,
haltOnFailure = True))
factory.addStep(DirectoryUpload(slavesrc="install_root", masterdest=masterdestLocation, url="binaries", compress="bz2"))
factory.addStep(DirectoryUpload(slavesrc="install_root", masterdest=master_loc_rel, url="binaries", compress="bz2"))
fixPermissions(factory)
return factory
@@ -547,7 +598,7 @@ def python_source_slave(key, platform, conda_env, cmake_args = [], cmake_env = {
# working_folder = "build/wrappers/DEB"
#
# # Create the factory to add the actions to
# factory = getBaseFactory(gitMode=gitMode)
# factory = getBaseFactory()
#
# factory.addStep(ShellCommand(command = ' '.join(["chmod","+x","package.bsh","&&", "./package.bsh"]),
# workdir= working_folder,
@@ -556,7 +607,7 @@ def python_source_slave(key, platform, conda_env, cmake_args = [], cmake_env = {
# factory.addStep(ShellCommand(command = ' '.join(["mv","*.deb","DEB/DEB"]),
# workdir= working_folder,
# haltOnFailure = True))
# factory.addStep(DirectoryUpload(slavesrc="wrappers/DEB/DEB", masterdest=masterdestLocation, url="binaries/DEB", compress="bz2"))
# factory.addStep(DirectoryUpload(slavesrc="wrappers/DEB/DEB", masterdest=master_loc_rel, url="binaries/DEB", compress="bz2"))
#
# return factory
@@ -566,7 +617,7 @@ def excel_slave(gitMode = 'incremental'):
working_folder = "build/build"
# Create the factory to add the actions to
factory = getBaseFactory(gitMode=gitMode)
factory = getBaseFactory()
#
factory.addStep(MakeDirectory(dir='build/bin/MicrosoftExcel', haltOnFailure = True))
factory.addStep(MakeDirectory(dir=working_folder+'/32bitDLL', haltOnFailure = True))
@@ -600,7 +651,7 @@ def excel_slave(gitMode = 'incremental'):
factory.addStep(ShellCommand(command=["copy", "wrappers\Excel\CoolProp.xla", "bin\MicrosoftExcel"], workdir = 'build', haltOnFailure = True))
factory.addStep(ShellCommand(command=["copy", "wrappers\Excel\TestExcel.xlsx", "bin\MicrosoftExcel"], workdir = 'build', haltOnFailure = True))
# Upload the files
factory.addStep(DirectoryUpload(slavesrc="bin",masterdest=masterdestLocation,url="MicrosoftExcel",compress="bz2"))
factory.addStep(DirectoryUpload(slavesrc="bin",masterdest=master_loc_rel,url="MicrosoftExcel",compress="bz2"))
fixPermissions(factory)
return factory
@@ -610,7 +661,7 @@ def smath_builder(gitMode = 'incremental'):
working_folder = "build/build"
# Create the factory to add the actions to
factory = getBaseFactory(gitMode=gitMode)
factory = getBaseFactory()
#
factory.addStep(MakeDirectory(dir='build/bin/SMath', haltOnFailure = True))
factory.addStep(MakeDirectory(dir=working_folder+'/32bitDLL', haltOnFailure = True))
@@ -647,7 +698,7 @@ def smath_builder(gitMode = 'incremental'):
factory.addStep(ShellCommand(command=["build_zip"], workdir = 'build\\wrappers\\SMath\\coolprop_wrapper', haltOnFailure = True))
factory.addStep(ShellCommand(command=["copy", "/Y", "wrappers\\SMath\\coolprop_wrapper\\coolprop_wrapper.7z", "bin\\SMath"], workdir = 'build', haltOnFailure = True))
# Upload the files
factory.addStep(DirectoryUpload(slavesrc="bin",masterdest=masterdestLocation,url="SMath",compress="bz2"))
factory.addStep(DirectoryUpload(slavesrc="bin",masterdest=master_loc_rel,url="SMath",compress="bz2"))
fixPermissions(factory)
return factory
@@ -657,14 +708,14 @@ def julia_builder(gitMode = 'incremental'):
working_folder = "build/build"
# Create the factory to add the actions to
factory = getBaseFactory(gitMode=gitMode)
factory = getBaseFactory()
# Clean the install_root
factory.addStep(RemoveDirectory(dir="build/install_root/Julia", haltOnFailure = False))
factory.addStep(MakeDirectory(dir='build/install_root/Julia', haltOnFailure = True))
# Copy other files
factory.addStep(ShellCommand(command=["cp", "wrappers/Julia/CoolProp.jl", "install_root/Julia/"], workdir = 'build', haltOnFailure = True))
# Upload the files - TODO: Is this the correct directory?
factory.addStep(DirectoryUpload(slavesrc="install_root",masterdest=masterdestLocation,url="binaries",compress="bz2"))
factory.addStep(DirectoryUpload(slavesrc="install_root",masterdest=master_loc_rel,url="binaries",compress="bz2"))
fixPermissions(factory)
return factory
@@ -690,7 +741,7 @@ def cmake_slave(mod_name, platform, gitMode = 'incremental', install = True, cma
working_folder = "build/build"
# Create the factory to add the actions to
factory = getBaseFactory(gitMode=gitMode)
factory = getBaseFactory()
#
factory.addStep(MakeDirectory(dir=working_folder, haltOnFailure = True))
factory.addStep(RemoveDirectory(dir="build/install_root", haltOnFailure = True))
@@ -709,7 +760,7 @@ def cmake_slave(mod_name, platform, gitMode = 'incremental', install = True, cma
if test:
factory.addStep(ShellCommand(command=["ctest", "--extra-verbose"] + ctest_args, workdir = working_folder, haltOnFailure = True, env = cmake_env))
if install:
factory.addStep(DirectoryUpload(slavesrc="install_root",masterdest=masterdestLocation,url="binaries",compress="bz2"))
factory.addStep(DirectoryUpload(slavesrc="install_root",masterdest=master_loc_rel,url="binaries",compress="bz2"))
fixPermissions(factory)
return factory
@@ -730,7 +781,7 @@ def swig_matlab_builder(platform, gitMode = 'incremental', build_args = [], cmak
return pre_path+cmd
# Create the factory to add the actions to
factory = getBaseFactory(gitMode=gitMode)
factory = getBaseFactory()
#
# Download files from nightly build for swig
factory.addStep(FileDownload(mastersrc = 'public_html/nightly/swig+MATLAB/'+platform+'/swig_MATLAB.7z',
@@ -750,7 +801,7 @@ def swig_matlab_builder(platform, gitMode = 'incremental', build_args = [], cmak
# Run simple integration test, but only on platforms other than windows
factory.addStep(ShellCommand(command=["ctest", "--extra-verbose"] + ctest_args, workdir = "build/build", haltOnFailure = True))
# Upload the files
factory.addStep(DirectoryUpload(slavesrc="install_root",masterdest=masterdestLocation,url="binaries",compress="bz2"))
factory.addStep(DirectoryUpload(slavesrc="install_root",masterdest=master_loc_rel,url="binaries",compress="bz2"))
fixPermissions(factory)
# Return the object
return factory
@@ -765,7 +816,7 @@ def swig_scilab_builder(platform, gitMode = 'incremental'):
working_folder = "build/build"
# Create the factory to add the actions to
factory = getBaseFactory(gitMode=gitMode)
factory = getBaseFactory()
#
# Download files from nightly build for swig
factory.addStep(FileDownload(mastersrc = 'public_html/nightly/swig+SCILAB/'+platform+'/swig_SCILAB.7z', slavedest = 'swig_SCILAB.7z', haltOnFailure = True))
@@ -788,14 +839,14 @@ def swig_scilab_builder(platform, gitMode = 'incremental'):
workdir= "build/build",
haltOnFailure = True))
# Upload the files
factory.addStep(DirectoryUpload(slavesrc="install_root",masterdest=masterdestLocation,url="binaries",compress="bz2"))
factory.addStep(DirectoryUpload(slavesrc="install_root",masterdest=master_loc_rel,url="binaries",compress="bz2"))
fixPermissions(factory)
return factory
def SWIG_MATLAB_bin_builder(platform, gitMode = 'incremental', windows = False):
# Create the factory to add the actions to
factory = getBaseFactory(gitMode=gitMode)
factory = getBaseFactory()
# Build SWIG with MATLAB
args = ["python","build_swig_matlab.py"]
if windows:
@@ -819,7 +870,7 @@ def SWIG_MATLAB_bin_builder(platform, gitMode = 'incremental', windows = False):
def SWIG_scilab_bin_builder(platform, gitMode = 'incremental', windows = False):
# Create the factory to add the actions to
factory = getBaseFactory(gitMode=gitMode)
factory = getBaseFactory()
# Build SWIG with scilab
args = ["python","build_swig_scilab.py"]
if windows:
@@ -850,7 +901,7 @@ def memory_sanitizer_builder(gitMode = 'incremental'):
commons = dict(workdir = 'build/asan', haltOnFailure = True, env = dict(CLANG_ROOT = CLANG_ROOT))
# Create the factory to add the actions to
factory = getBaseFactory(gitMode=gitMode)
factory = getBaseFactory()
factory.addStep(MakeDirectory(dir='build/asan', haltOnFailure = True))
factory.addStep(ShellCommand(command='cmake .. -DCOOLPROP_CLANG_ADDRESS_SANITIZER=ON -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_C_COMPILER=${CLANG_ROOT}/bin/clang -DCMAKE_CXX_COMPILER=${CLANG_ROOT}/bin/clang++', **commons))
factory.addStep(ShellCommand(command='DYLD_LIBRARY_PATH=${CLANG_ROOT}/lib/clang/3.5.0/lib/darwin/ ASAN_SYMBOLIZER_PATH=${CLANG_ROOT}/bin/llvm-symbolizer ASAN_OPTIONS=verbosity=1 cmake --build .', **commons))
@@ -864,7 +915,7 @@ def vxworks_module_builder(gitMode = 'incremental', cmake_args = [], cmake_env =
working_folder = "build/build"
# Create the factory to add the actions to
factory = getBaseFactory(gitMode=gitMode)
factory = getBaseFactory()
# Directory for build
factory.addStep(MakeDirectory(dir='build/build', haltOnFailure = True))
# Generate makefile
@@ -872,7 +923,7 @@ def vxworks_module_builder(gitMode = 'incremental', cmake_args = [], cmake_env =
# Build
factory.addStep(ShellCommand(command = 'cmake --build . --target install', workdir = 'build/build', haltOnFailure = True))
# Upload the files
factory.addStep(DirectoryUpload(slavesrc="install_root", masterdest=masterdestLocation,url="binaries",compress="bz2"))
factory.addStep(DirectoryUpload(slavesrc="install_root", masterdest=master_loc_rel,url="binaries",compress="bz2"))
fixPermissions(factory)
return factory

View File

@@ -20,6 +20,12 @@ function git_pull {
fi
popd
}
function git_cfg {
pushd /home/$USER/buildbot/CoolProp.git
git reset HEAD dev/buildbot/master/master.cfg
git checkout -- dev/buildbot/master/master.cfg
popd
}
function stop {
buildbot stop /home/$USER/buildbot/server-master/
}
@@ -39,6 +45,11 @@ elif [ "$CMD" = "reconfig" ]; then
git_pull
clean
reconfig
elif [ "$CMD" = "reconfigmaster" ]; then
git_cfg
git_pull
clean
reconfig
elif [ "$CMD" = "start" ]; then
git_pull
clean

View File

@@ -184,19 +184,19 @@ def remove_readonly(func, path, _):
try: func(path)
except: remove_all(path); pass
#
cmd = ['conda','remove','--all','-yq','-n']
for t in ['_test','_build']:
try:
subprocess.check_call(cmd+[t], stdout=sys.stdout, stderr=sys.stderr)
except:
envs = run_command(['conda','env','list'])[0].decode("utf-8").splitlines()
for env in envs:
lst = env.split(' ')
if len(lst)>0 and lst[0] == t:
dir = ' '.join(lst[1:]).strip()
print("Manually removing: "+dir)
shutil.rmtree(dir, onerror=remove_readonly)
pass
#cmd = ['conda','remove','--all','-yq','-n']
#for t in ['_test','_build']:
# try:
# subprocess.check_call(cmd+[t], stdout=sys.stdout, stderr=sys.stderr)
# except:
# envs = run_command(['conda','env','list'])[0].decode("utf-8").splitlines()
# for env in envs:
# lst = env.split(' ')
# if len(lst)>0 and lst[0] == t:
# dir = ' '.join(lst[1:]).strip()
# print("Manually removing: "+dir)
# shutil.rmtree(dir, onerror=remove_readonly)
# pass
tar = os.path.abspath(os.path.join(os.path.dirname(__file__),'conda')).strip()
if os.path.isdir(tar): shutil.rmtree(tar, onerror=remove_readonly)
ver = sys.version_info