From cb952a03fb89c29bfd2ce12c1a9128ea7e16170d Mon Sep 17 00:00:00 2001 From: Jorrit Wronski Date: Sun, 11 Oct 2015 16:12:59 +0200 Subject: [PATCH] Added web builder --- Web/develop/buildbot.rst | 15 ++++++++------- dev/buildbot/master/master.cfg | 4 ++-- dev/docker/Dockerfile.generator.py | 25 +++++++++++++++++++++++++ dev/docker/Dockerfile.slave.web.tpl | 18 ++++++++++++++++++ dev/docker/slavebase/Dockerfile | 2 +- dev/docker/slavepython/Dockerfile | 2 +- dev/docker/slaveweb/Dockerfile | 27 +++++++++++++++++++++++++++ 7 files changed, 82 insertions(+), 11 deletions(-) create mode 100644 dev/docker/Dockerfile.slave.web.tpl create mode 100644 dev/docker/slaveweb/Dockerfile diff --git a/Web/develop/buildbot.rst b/Web/develop/buildbot.rst index fba95ef2..2ee1d055 100644 --- a/Web/develop/buildbot.rst +++ b/Web/develop/buildbot.rst @@ -482,7 +482,7 @@ Documentation Builds Some parts of the documentation are quite involved. That is why we decided not to rebuild the whole documentation after every commit. There is a special python -script that runs a day and performs the most expensive jobs during +script that runs once a day and performs the most expensive jobs during documentation rebuild. This covers the generation of validation figures for all fluids and the fitting reports for the incompressible fluids. @@ -514,14 +514,15 @@ build system: * You can then run the official coolprop buildbot configuration with:: - docker run --env-file ./Dockerfile.slave.env.list --name="slavename" coolprop/slavebase + docker run --env-file ./Dockerfile.slave.env.list --name="${SLAVENAME}" coolprop/slavebase -* Some steps require the upload of files to teh servers. In such cases, you - should copy your SSH configuration to the container to make use of the - automatic login that is required for rsync to work properly:: +* Some steps require the upload of files to different servers. In such cases, you + should copy your SSH configuration or other login information to the container to + make use of the automatic login that is required for rsync to work properly:: - docker cp ${HOME}/.ssh slavename:/home/buildbot/ - docker exec slavename chown -R buildbot /home/buildbot/.ssh + docker cp ${HOME}/.ssh ${SLAVENAME}:/home/buildbot/ + docker cp ${HOME}/.pypirc ${SLAVENAME}:/home/buildbot/ + docker exec ${SLAVENAME} chown -R buildbot /home/buildbot/ .. note:: If you cannot copy the SSH keys, you can change the upload function in the diff --git a/dev/buildbot/master/master.cfg b/dev/buildbot/master/master.cfg index 2444567e..f969c787 100644 --- a/dev/buildbot/master/master.cfg +++ b/dev/buildbot/master/master.cfg @@ -1040,7 +1040,7 @@ windowsDTUslave.pyins[getIDstr("windows", "64bit", "py34")] = tmpins + ['--cmake # linuxJorritSlave.pyins[getIDstr( "linux" , "64bit", 0 )] = baseins osxIPUslave = PythonSlaveConfig("OSX-IPU-slave") -osxIPUslave.pyact[getIDstr( "osx" , "64bit", 0 )] = "source /Users/jowr/miniconda/bin/activate" +osxIPUslave.pyact[getIDstr( "osx" , "64bit", 0 )] = "source activate" osxIPUslave.pyenv[getIDstr( "osx" , "64bit", "py27")] = "CoolProp27" osxIPUslave.pyenv[getIDstr( "osx" , "64bit", "py33")] = "CoolProp33" osxIPUslave.pyenv[getIDstr( "osx" , "64bit", "py34")] = "CoolProp34" @@ -1061,7 +1061,7 @@ l64IPUslave.pyenv[getIDstr( "linux" , "64bit", "py34")] = "CoolProp34" l64IPUslave.pyins[getIDstr( "linux" , 0 , 0 )] = baseins l32IPUslave = PythonSlaveConfig("Linux32-IPU-slave") -l32IPUslave.pyact[getIDstr( "linux" , "32bit", 0 )] = "source /home/jowr/miniconda3/bin/activate" +l32IPUslave.pyact[getIDstr( "linux" , "32bit", 0 )] = "source activate" l32IPUslave.pyenv[getIDstr( "linux" , "32bit", "py27")] = "CoolProp27" l32IPUslave.pyenv[getIDstr( "linux" , "32bit", "py33")] = "CoolProp33" l32IPUslave.pyenv[getIDstr( "linux" , "32bit", "py34")] = "CoolProp34" diff --git a/dev/docker/Dockerfile.generator.py b/dev/docker/Dockerfile.generator.py index 083c9a7d..acb6700f 100644 --- a/dev/docker/Dockerfile.generator.py +++ b/dev/docker/Dockerfile.generator.py @@ -50,6 +50,20 @@ cnd_run_pkgs = ["numpy", "scipy", "matplotlib", "pandas"] # 4) Known hosts #ssh_hosts = ["bitbucket.org", "github.com", "coolprop.dreamhosters.com", "coolprop.org"] +# 5) Web packages: +lin_web_pkgs = ["build-essential", "git", "gcc", "gfortran", "cmake"] +lin_web_pkgs += ["python-dev", "libxml2-dev", "libxslt1-dev", "libxslt1.1", "doxygen", "libtool"] +lin_web_pkgs += ["libatlas3-base", "libblas3", "liblapack3"] +lin_web_pkgs += ["libatlas-dev", "libblas-dev", "liblapack-dev"] +lin_web_pkgs += ["python-dev", "libpng-dev", "tk", "libfreetype6-dev"] + +cnd_web_pkgs = ["setuptools", "cython"] +cnd_web_pkgs += ["numpy", "scipy", "matplotlib", "pandas"] +cnd_web_pkgs += ["ipython", "wxpython"] + +pip_web_pkgs = ["sphinx", "cloud-sptheme"] +pip_web_pkgs += ["sphinxcontrib-bibtex", "sphinxcontrib-doxylink", "sphinxcontrib-napoleon"] + # local_dict = dict( author = author, @@ -60,6 +74,9 @@ local_dict = dict( cnd_env_pyt = [str(x[0])+" "+str(x[1]) for x in zip(cnd_env,cnd_pyt)], cnd_env = cnd_env, cnd_dev_pkgs = cnd_dev_pkgs+cnd_run_pkgs, + lin_web_pkgs = lin_web_pkgs, + cnd_web_pkgs = cnd_web_pkgs, + pip_web_pkgs = pip_web_pkgs, ) # template_path = 'Dockerfile.slave.base.tpl' @@ -75,6 +92,14 @@ f = codecs.open(os.path.join(tar_dir,'slavepython','Dockerfile'),mode='wb',encod f.write(tpl_first_line.format("# "+tpl_mtime_line,template_path)) f.write(template.render(**local_dict)) f.close() +# +template_path = 'Dockerfile.slave.web.tpl' +template = environment.get_template(template_path) +f = codecs.open(os.path.join(tar_dir,'slaveweb','Dockerfile'),mode='wb',encoding='utf-8') +f.write(tpl_first_line.format("# "+tpl_mtime_line,template_path)) +f.write(template.render(**local_dict)) +f.close() + print(r""" diff --git a/dev/docker/Dockerfile.slave.web.tpl b/dev/docker/Dockerfile.slave.web.tpl new file mode 100644 index 00000000..d20603b0 --- /dev/null +++ b/dev/docker/Dockerfile.slave.web.tpl @@ -0,0 +1,18 @@ +# Set the base image to Debian +FROM coolprop/slavepython +# File Author / Maintainer +MAINTAINER {{ author }} ({{ email }}) +# +RUN apt-get update -qq && \ + apt-get install -y{% for pkg in lin_web_pkgs %} {{ pkg }}{% endfor %} && \ + apt-get clean +# +RUN \{% for py in cnd_env_pyt %} +conda install -n {{ py }}{% for pkg in cnd_web_pkgs %} {{ pkg }}{% endfor %} && \{% endfor %} +conda clean -yilts +# +RUN \{% for py in cnd_env %} +source activate {{ py }} && \ +pip install{% for pkg in pip_web_pkgs %} {{ pkg }}{% endfor %} && \ +source deactivate && \{% endfor %} +conda clean -yilts diff --git a/dev/docker/slavebase/Dockerfile b/dev/docker/slavebase/Dockerfile index 60f68a4d..d7d72ae0 100644 --- a/dev/docker/slavebase/Dockerfile +++ b/dev/docker/slavebase/Dockerfile @@ -1,4 +1,4 @@ -# 2015-10-11 13:42 CAUTION: This file is automatically generated from Dockerfile.slave.base.tpl, do not edit it manually. +# 2015-10-11 16:07 CAUTION: This file is automatically generated from Dockerfile.slave.base.tpl, do not edit it manually. # Set the base image to Debian FROM debian:stable # File Author / Maintainer diff --git a/dev/docker/slavepython/Dockerfile b/dev/docker/slavepython/Dockerfile index 46fb6332..b69106e4 100644 --- a/dev/docker/slavepython/Dockerfile +++ b/dev/docker/slavepython/Dockerfile @@ -1,4 +1,4 @@ -# 2015-10-11 13:42 CAUTION: This file is automatically generated from Dockerfile.slave.python.tpl, do not edit it manually. +# 2015-10-11 16:07 CAUTION: This file is automatically generated from Dockerfile.slave.python.tpl, do not edit it manually. # Set the base image to Debian FROM coolprop/slavebase # File Author / Maintainer diff --git a/dev/docker/slaveweb/Dockerfile b/dev/docker/slaveweb/Dockerfile new file mode 100644 index 00000000..b53842f0 --- /dev/null +++ b/dev/docker/slaveweb/Dockerfile @@ -0,0 +1,27 @@ +# 2015-10-11 16:07 CAUTION: This file is automatically generated from Dockerfile.slave.web.tpl, do not edit it manually. +# Set the base image to Debian +FROM coolprop/slavepython +# File Author / Maintainer +MAINTAINER Jorrit Wronski (jowr@ipu.dk) +# +RUN apt-get update -qq && \ + apt-get install -y build-essential git gcc gfortran cmake python-dev libxml2-dev libxslt1-dev libxslt1.1 doxygen libtool libatlas3-base libblas3 liblapack3 libatlas-dev libblas-dev liblapack-dev python-dev libpng-dev tk libfreetype6-dev && \ + apt-get clean +# +RUN \ +conda install -n CoolProp27 python=2.7 setuptools cython numpy scipy matplotlib pandas ipython wxpython && \ +conda install -n CoolProp33 python=3.3 setuptools cython numpy scipy matplotlib pandas ipython wxpython && \ +conda install -n CoolProp34 python=3.4 setuptools cython numpy scipy matplotlib pandas ipython wxpython && \ +conda clean -yilts +# +RUN \ +source activate CoolProp27 && \ +pip install sphinx cloud-sptheme sphinxcontrib-bibtex sphinxcontrib-doxylink sphinxcontrib-napoleon && \ +source deactivate && \ +source activate CoolProp33 && \ +pip install sphinx cloud-sptheme sphinxcontrib-bibtex sphinxcontrib-doxylink sphinxcontrib-napoleon && \ +source deactivate && \ +source activate CoolProp34 && \ +pip install sphinx cloud-sptheme sphinxcontrib-bibtex sphinxcontrib-doxylink sphinxcontrib-napoleon && \ +source deactivate && \ +conda clean -yilts \ No newline at end of file