diff --git a/src/chimp/streamline.F90 b/src/chimp/streamline.F90 index 73e5c71e..4126ed7f 100644 --- a/src/chimp/streamline.F90 +++ b/src/chimp/streamline.F90 @@ -12,6 +12,7 @@ module streamline real(rp), private :: ShueScl = 2.0 !Safety factor for Shue MP real(rp), private :: rShue = 6.0 !Radius to start checking Shue integer , private :: NpChk = 10 !Cadence for Shue checking + logical , private :: doShueG = .false. !Global doShue, can be overriden by optional arguments to routines contains @@ -21,15 +22,16 @@ module streamline if (Model%isMAGE .and. (trim(toUpper(Model%uID)) == "EARTHCODE")) then !This is for Earth and we're running in tandem w/ mage - !Setup shue for short-circuiting - write(*,*) "Initializing SHUE-MP checking ..." call inpXML%Set_Val(ShueScl,'streamshue/ShueScl' ,ShueScl) call inpXML%Set_Val(rShue ,'streamshue/rShue' ,rShue ) call inpXML%Set_Val(NpChk ,'streamshue/NpChk' ,NpChk ) + call inpXML%Set_Val(doShueG,'streamshue/doShue' ,.false.) + if (doShueG) write(*,*) "Initializing SHUE-MP checking ..." else - !Otherwise don't care about Shue rShue = HUGE + doShueG = .false. endif + end subroutine setShue !Trace field line w/ seed point x0 (both directions) @@ -84,10 +86,11 @@ module streamline doSH = .false. endif + !If optional argument, do whatever it says. Otherwise default to global value if (present(doShueO)) then doShue = doShueO else - doShue = .false. + doShue = doShueG endif !Allocate temp arrays to hold information along each direction @@ -749,7 +752,7 @@ module streamline if (present(doShueO)) then doShue = doShueO else - doShue = .false. + doShue = doShueG endif !Initialize @@ -827,7 +830,7 @@ module streamline !Slimmed down projection to northern hemisphere for MAGE !RinO is optional cut-off inner radius when in northern hemisphere !epsO is optional epsilon (otherwise use Model default) - subroutine mageproject(Model,ebState,x0,t,xyz,Np,isG,epsO,MaxStepsO) + subroutine mageproject(Model,ebState,x0,t,xyz,Np,isG,epsO,MaxStepsO,doShueO) type(chmpModel_T), intent(in) :: Model type(ebState_T), intent(in) :: ebState real(rp), intent(in) :: x0(NDIM),t @@ -836,12 +839,13 @@ module streamline logical, intent(out) :: isG real(rp), intent(in), optional :: epsO integer , intent(in), optional :: MaxStepsO + logical , intent(in), optional :: doShueO type(GridPoint_T) :: gPt integer :: sgn,MaxSteps real(rp) :: eps,h real(rp), dimension(NDIM) :: dx,B,oB - logical :: inDom,isSC,isDone + logical :: inDom,isSC,isDone,doShue if (present(epsO)) then eps = epsO @@ -855,6 +859,12 @@ module streamline MaxSteps = MaxFL endif + if (present(doShueO)) then + doShue = doShueO + else + doShue = doShueG + endif + sgn = +1 !Step towards NH isG = .false. !Initialize @@ -935,7 +945,7 @@ module streamline logical :: inMP inDom = inDomain(xyz,Model,ebState%ebGr) - if ( (modulo(Np,NpChk) == 0) .and. (norm2(gPt%xyz)>=rShue) ) then + if ( doShue .and. (modulo(Np,NpChk) == 0) .and. (norm2(gPt%xyz)>=rShue) ) then inMP = inShueMP_SM(xyz,ShueScl) else inMP = .true. diff --git a/src/raiju/raijuIO.F90 b/src/raiju/raijuIO.F90 index d4d82f99..79813862 100644 --- a/src/raiju/raijuIO.F90 +++ b/src/raiju/raijuIO.F90 @@ -275,7 +275,7 @@ module raijuIO call AddOutVar(IOVars,"pot_corot",State%pot_corot(is:ie+1,js:je+1),uStr="kV",dStr="(corners) Corotation potential") ! Idk about you but I did not expect true to equal -1 - allocate(outTmp2D(is:ie, Grid%Nk)) + allocate(outTmp2D(Grid%shGrid%isg:Grid%shGrid%ieg, Grid%Nk)) where (State%activeShells) outTmp2D = 1.0 elsewhere @@ -421,6 +421,7 @@ module raijuIO allocate(outTmp2D(Grid%shGrid%isg:Grid%shGrid%ieg ,Grid%shGrid%jsg:Grid%shGrid%jeg)) call calcMapJacNorm(Grid, State%xyzMin, outTmp2D) call AddOutVar(IOVars, "mapJacNorm", outTmp2D(is:ie,js:je), dStr="L_(2,1) norm of lat/lon => xyzMin Jacobian") + deallocate(outTmp2D) endif call WriteVars(IOVars,.true.,Model%raijuH5, gStr) diff --git a/src/voltron/ebsquish.F90 b/src/voltron/ebsquish.F90 index 4ae993d4..d3806ec9 100644 --- a/src/voltron/ebsquish.F90 +++ b/src/voltron/ebsquish.F90 @@ -422,8 +422,8 @@ module ebsquish return endif - !Do quick short-cut to save us some effort - isGood = inShueMP_SM(xyz,ShueScl) + !Can add short circuit code here, but nothing for now + isGood = .true. !Let's be optimistic if (.not. isGood) return ! trap for when we're within epsilon of the inner boundary diff --git a/src/voltron/innermagsphere.F90 b/src/voltron/innermagsphere.F90 index fdb509d1..a7fcda95 100644 --- a/src/voltron/innermagsphere.F90 +++ b/src/voltron/innermagsphere.F90 @@ -74,7 +74,9 @@ module innermagsphere !call vApp%imagApp%doInit(iXML,gApp%Model%isRestart,vApp) call vApp%imagApp%InitModel(iXML) - call vApp%imagApp%InitIO(iXML) + if(vApp%writeFiles) then + call vApp%imagApp%InitIO(iXML) + endif end subroutine InitInnerMag diff --git a/src/voltron/mpi/voltapp_mpi.F90 b/src/voltron/mpi/voltapp_mpi.F90 index c8e7e12b..82653045 100644 --- a/src/voltron/mpi/voltapp_mpi.F90 +++ b/src/voltron/mpi/voltapp_mpi.F90 @@ -152,8 +152,8 @@ module voltapp_mpi deallocate(vApp%ebTrcApp%ebSquish%blockStartIndices) allocate(vApp%ebTrcApp%ebSquish%blockStartIndices(vApp%ebTrcApp%ebSquish%numSquishBlocks)) do b=1,vApp%ebTrcApp%ebSquish%numSquishBlocks - vApp%ebTrcApp%ebSquish%blockStartIndices(b) = vApp%ebTrcApp%ebState%ebGr%ks + & - ((b-1)*(vApp%ebTrcApp%ebState%ebGr%ke+1))/vApp%ebTrcApp%ebSquish%numSquishBlocks + vApp%ebTrcApp%ebSquish%blockStartIndices(b) = & + GetAdjustedSquishStart(vApp,((b-1)*(vApp%ebTrcApp%ebState%ebGr%ke+1))/vApp%ebTrcApp%ebSquish%numSquishBlocks) enddo endif call createLoadBalancer(vApp%squishLb, nHelpers,& @@ -443,7 +443,7 @@ module voltapp_mpi vApp%deepProcessingInProgress = .true. call Toc("DeepUpdate", .true.) - else + elseif(vApp%doDeep) then vApp%gApp%Grid%Gas0 = 0 !Load TM03 into Gas0 for ingestion during spinup !Note: Using vApp%time instead of gamera time units diff --git a/src/voltron/voltCplHelper.F90 b/src/voltron/voltCplHelper.F90 index b92b6467..4a685b67 100644 --- a/src/voltron/voltCplHelper.F90 +++ b/src/voltron/voltCplHelper.F90 @@ -64,7 +64,7 @@ module voltCplHelper call CleanLine(magLine) !Note: Not using volt time b/c chimp wants time in its units call genLine(ebApp%ebModel,ebApp%ebState,xyz0,ebApp%ebState%eb1%time, magLine,& - doShueO=.false.,doNHO=doNH,doSHO=doSH) + doNHO=doNH,doSHO=doSH) call Line2Tube(ebApp,vApp%planet,magLine,vApp%State%ijTubes(i,j)) endif diff --git a/src/voltron/voltapp.F90 b/src/voltron/voltapp.F90 index b277f489..cbce3f27 100644 --- a/src/voltron/voltapp.F90 +++ b/src/voltron/voltapp.F90 @@ -719,7 +719,8 @@ module voltapp call inpXML%Set_Val(Model%epsds,'tracer/epsds',1.0e-2) call setBackground(Model,inpXML) call inpXML%Set_Val(Model%doDip,'tracer/doDip',.false.) - + call setStreamline(Model,inpXML) + !Initialize ebState if (gApp%Model%doMultiF) then write(*,*) "Initializing MF-Chimp ..." @@ -736,7 +737,8 @@ module voltapp !Initialize squish indices allocate(vApp%ebTrcApp%ebSquish%blockStartIndices(vApp%ebTrcApp%ebSquish%numSquishBlocks)) do b=1,vApp%ebTrcApp%ebSquish%numSquishBlocks - vApp%ebTrcApp%ebSquish%blockStartIndices(b) = ebGr%ks + ((b-1)*(ebGr%ke+1))/vApp%ebTrcApp%ebSquish%numSquishBlocks + vApp%ebTrcApp%ebSquish%blockStartIndices(b) = & + GetAdjustedSquishStart(vApp,((b-1)*(ebGr%ke+1))/vApp%ebTrcApp%ebSquish%numSquishBlocks) enddo !Do simple test to make sure locator is reasonable diff --git a/testingScripts/intelCheckSubmitReport-template.pbs b/testingScripts/intelCheckSubmitReport-template.pbs index c859e16b..381e3c64 100644 --- a/testingScripts/intelCheckSubmitReport-template.pbs +++ b/testingScripts/intelCheckSubmitReport-template.pbs @@ -19,19 +19,29 @@ module load {{ module }} module list echo 'Loading python environment.' -__conda_setup="$('/glade/u/home/ewinter/miniconda3/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" -if [ $? -eq 0 ]; then - eval "$__conda_setup" -else - if [ -f "/glade/u/home/ewinter/miniconda3/etc/profile.d/conda.sh" ]; then - . "/glade/u/home/ewinter/miniconda3/etc/profile.d/conda.sh" +if [ -d "${mage_test_root}/miniconda3" ]; then + echo 'Loading local miniconda3' + mage_test_root='{{ mage_test_root }}' + export CONDARC="${mage_test_root}/.condarc" + export CONDA_ENVS_PATH="${mage_test_root}/.conda" + mage_miniconda3="${mage_test_root}/miniconda3" + mage_conda="${mage_miniconda3}/bin/conda" + __conda_setup="$($mage_conda 'shell.bash' 'hook' 2> /dev/null)" + if [ $? -eq 0 ]; then + eval "$__conda_setup" else - export PATH="/glade/u/home/ewinter/miniconda3/bin:$PATH" + if [ -f "$mage_miniconda3/etc/profile.d/conda.sh" ]; then + . "$mage_miniconda3/etc/profile.d/conda.sh" + else + export PATH="$mage_miniconda3/bin:$PATH" + fi fi + unset __conda_setup +else + echo 'Loading conda module' + module load conda fi -unset __conda_setup - -conda activate kaiju-3.12-testing +conda activate {{ conda_environment }} echo 'Setting up MAGE environment.' source {{ kaijuhome }}/scripts/setupEnvironment.sh diff --git a/testingScripts/intelChecks.py b/testingScripts/intelChecks.py index 48cb0ae6..e7c2703f 100644 --- a/testingScripts/intelChecks.py +++ b/testingScripts/intelChecks.py @@ -85,8 +85,7 @@ BUILD_BIN_DIR = "bin" # Data and configuration files used by the Intel Inspector tests. TEST_INPUT_FILES = [ - "bcwind.h5", - "lfmD.h5", + "tinyCase.xml", "memSuppress.sup", "rcmconfig.h5", "threadSuppress.sup", @@ -309,6 +308,66 @@ def intelChecks(args: dict): from_path = os.path.join(TEST_INPUTS_DIRECTORY, filename) to_path = os.path.join(".", filename) shutil.copyfile(from_path, to_path) + + # Generate bcwind data file. + if verbose: + print("Creating bcwind data file.") + cmd = "cda2wind.py -t0 2016-08-09T09:00:00 -t1 2016-08-09T11:00:00" + if debug: + print(f"cmd = {cmd}") + try: + cproc = subprocess.run(cmd, shell=True, check=True) + except subprocess.CalledProcessError as e: + print("ERROR: Unable to create bcwind data file for module set " + f"{module_set_name}.\n" + f"e.cmd = {e.cmd}\n" + f"e.returncode = {e.returncode}\n" + "See testing log for output from cda2wind.py.\n" + "Skipping remaining steps for module set" + f"{module_set_name}\n") + continue + if debug: + print(f"cproc = {cproc}") + + # Generate the LFM grid file. + if verbose: + print("Creating LFM grid file.") + cmd = "genLFM.py -gid D" + if debug: + print(f"cmd = {cmd}") + try: + cproc = subprocess.run(cmd, shell=True, check=True) + except subprocess.CalledProcessError as e: + print("ERROR: Unable to create LFM grid file for module set " + f"{module_set_name}.\n" + f"e.cmd = {e.cmd}\n" + f"e.returncode = {e.returncode}\n" + "See testing log for output from genLFM.py.\n" + "Skipping remaining steps for module set" + f"{module_set_name}\n") + continue + if debug: + print(f"cproc = {cproc}") + + # Generate the Raiju configuration file. + if verbose: + print("Creating Raiju configuration file.") + cmd = "genRAIJU.py" + if debug: + print(f"cmd = {cmd}") + try: + cproc = subprocess.run(cmd, shell=True, check=True) + except subprocess.CalledProcessError as e: + print("ERROR: Unable to create Raiju configuration file" + f" for module set {module_set_name}.\n" + f"e.cmd = {e.cmd}\n" + f"e.returncode = {e.returncode}\n" + "See testing log for output from genRAIJU.py.\n" + "Skipping remaining steps for module set " + f"{module_set_name}\n") + continue + if debug: + print(f"cproc = {cproc}") # -------------------------------------------------------------------- diff --git a/testingScripts/relCasePost-template.pbs b/testingScripts/relCasePost-template.pbs index 2f968271..31deeb57 100644 --- a/testingScripts/relCasePost-template.pbs +++ b/testingScripts/relCasePost-template.pbs @@ -12,23 +12,29 @@ echo "Job $PBS_JOBID started at `date` on `hostname` in directory `pwd`." echo 'Loading python environment.' -mage_test_root='{{ mage_test_root }}' -export CONDARC="${mage_test_root}/.condarc" -export CONDA_ENVS_PATH="${mage_test_root}/.conda" -mage_miniconda3="${mage_test_root}/miniconda3" -mage_conda="${mage_miniconda3}/bin/conda" -__conda_setup="$($mage_conda 'shell.bash' 'hook' 2> /dev/null)" -if [ $? -eq 0 ]; then - eval "$__conda_setup" -else - if [ -f "$mage_miniconda3/etc/profile.d/conda.sh" ]; then - . "$mage_miniconda3/etc/profile.d/conda.sh" +if [ -d "${mage_test_root}/miniconda3" ]; then + echo 'Loading local miniconda3' + mage_test_root='{{ mage_test_root }}' + export CONDARC="${mage_test_root}/.condarc" + export CONDA_ENVS_PATH="${mage_test_root}/.conda" + mage_miniconda3="${mage_test_root}/miniconda3" + mage_conda="${mage_miniconda3}/bin/conda" + __conda_setup="$($mage_conda 'shell.bash' 'hook' 2> /dev/null)" + if [ $? -eq 0 ]; then + eval "$__conda_setup" else - export PATH="$mage_miniconda3/bin:$PATH" + if [ -f "$mage_miniconda3/etc/profile.d/conda.sh" ]; then + . "$mage_miniconda3/etc/profile.d/conda.sh" + else + export PATH="$mage_miniconda3/bin:$PATH" + fi fi + unset __conda_setup +else + echo 'Loading conda module' + module load conda fi -unset __conda_setup -conda activate kaiju-3.8-testing +conda activate {{ conda_environment }} #echo 'Setting up MAGE environment.' source {{ kaijuhome }}/scripts/setupEnvironment.sh diff --git a/testingScripts/relCaseReport-template.pbs b/testingScripts/relCaseReport-template.pbs index 60b60d5e..0c817377 100644 --- a/testingScripts/relCaseReport-template.pbs +++ b/testingScripts/relCaseReport-template.pbs @@ -12,23 +12,29 @@ echo "Job $PBS_JOBID started at `date` on `hostname` in directory `pwd`." echo 'Loading python environment.' -mage_test_root='{{ mage_test_root }}' -export CONDARC="${mage_test_root}/.condarc" -export CONDA_ENVS_PATH="${mage_test_root}/.conda" -mage_miniconda3="${mage_test_root}/miniconda3" -mage_conda="${mage_miniconda3}/bin/conda" -__conda_setup="$($mage_conda 'shell.bash' 'hook' 2> /dev/null)" -if [ $? -eq 0 ]; then - eval "$__conda_setup" -else - if [ -f "$mage_miniconda3/etc/profile.d/conda.sh" ]; then - . "$mage_miniconda3/etc/profile.d/conda.sh" +if [ -d "${mage_test_root}/miniconda3" ]; then + echo 'Loading local miniconda3' + mage_test_root='{{ mage_test_root }}' + export CONDARC="${mage_test_root}/.condarc" + export CONDA_ENVS_PATH="${mage_test_root}/.conda" + mage_miniconda3="${mage_test_root}/miniconda3" + mage_conda="${mage_miniconda3}/bin/conda" + __conda_setup="$($mage_conda 'shell.bash' 'hook' 2> /dev/null)" + if [ $? -eq 0 ]; then + eval "$__conda_setup" else - export PATH="$mage_miniconda3/bin:$PATH" + if [ -f "$mage_miniconda3/etc/profile.d/conda.sh" ]; then + . "$mage_miniconda3/etc/profile.d/conda.sh" + else + export PATH="$mage_miniconda3/bin:$PATH" + fi fi + unset __conda_setup +else + echo 'Loading conda module' + module load conda fi -unset __conda_setup -conda activate kaiju-3.8-testing +conda activate {{ conda_environment }} #echo 'Setting up MAGE environment.' source {{ kaijuhome }}/scripts/setupEnvironment.sh diff --git a/testingScripts/relativeCase-template.pbs b/testingScripts/relativeCase-template.pbs index dfedad67..24401e13 100644 --- a/testingScripts/relativeCase-template.pbs +++ b/testingScripts/relativeCase-template.pbs @@ -23,23 +23,29 @@ module load {{ module }} module list echo 'Loading python environment.' -mage_test_root='{{ mage_test_root }}' -export CONDARC="${mage_test_root}/.condarc" -export CONDA_ENVS_PATH="${mage_test_root}/.conda" -mage_miniconda3="${mage_test_root}/miniconda3" -mage_conda="${mage_miniconda3}/bin/conda" -__conda_setup="$($mage_conda 'shell.bash' 'hook' 2> /dev/null)" -if [ $? -eq 0 ]; then - eval "$__conda_setup" -else - if [ -f "$mage_miniconda3/etc/profile.d/conda.sh" ]; then - . "$mage_miniconda3/etc/profile.d/conda.sh" +if [ -d "${mage_test_root}/miniconda3" ]; then + echo 'Loading local miniconda3' + mage_test_root='{{ mage_test_root }}' + export CONDARC="${mage_test_root}/.condarc" + export CONDA_ENVS_PATH="${mage_test_root}/.conda" + mage_miniconda3="${mage_test_root}/miniconda3" + mage_conda="${mage_miniconda3}/bin/conda" + __conda_setup="$($mage_conda 'shell.bash' 'hook' 2> /dev/null)" + if [ $? -eq 0 ]; then + eval "$__conda_setup" else - export PATH="$mage_miniconda3/bin:$PATH" + if [ -f "$mage_miniconda3/etc/profile.d/conda.sh" ]; then + . "$mage_miniconda3/etc/profile.d/conda.sh" + else + export PATH="$mage_miniconda3/bin:$PATH" + fi fi + unset __conda_setup +else + echo 'Loading conda module' + module load conda fi -unset __conda_setup -conda activate kaiju-3.8-testing +conda activate {{ conda_environment }} echo 'Setting up MAGE environment.' source {{ kaijuhome }}/scripts/setupEnvironment.sh diff --git a/testingScripts/relativeTests.py b/testingScripts/relativeTests.py index 07c59df1..5cdd5f7c 100755 --- a/testingScripts/relativeTests.py +++ b/testingScripts/relativeTests.py @@ -160,7 +160,7 @@ def generateAndRunCase(caseName,pbsTemplate,pbs_options,xmlTemplate,xml_options, shutil.copy2('../voltron.x', './voltron.x') shutil.copy2('../voltron_mpi.x', './voltron_mpi.x') shutil.copy2('../lfmD.h5', './lfmD.h5') - shutil.copy2('../rcmconfig.h5', './rcmconfig.h5') + shutil.copy2('../raijuconfig.h5', './raijuconfig.h5') shutil.copy2('../bcwind.h5', './bcwind.h5') # Submit the job if verbose: @@ -643,16 +643,16 @@ def main(): f"{module_set_name}\n") continue - # Generate the RCM configuration file. + # Generate the Raiju configuration file. if verbose: - print('Creating RCM configuration file.') - cmd = 'genRCM.py' + print('Creating Raiju configuration file.') + cmd = 'genRAIJU.py' if debug: print(f"cmd = {cmd}") try: _ = subprocess.run(cmd, shell=True, check=True) except subprocess.CalledProcessError as e: - print('ERROR: Unable to create RCM configuration file' + print('ERROR: Unable to create Raiju configuration file' f" for module set {module_set_name}.\n" f"e.cmd = {e.cmd}\n" f"e.returncode = {e.returncode}\n" @@ -674,6 +674,7 @@ def main(): base_pbs_options['mage_test_root'] = os.environ['MAGE_TEST_ROOT'] base_pbs_options['mage_test_set_root'] = os.environ['MAGE_TEST_SET_ROOT'] base_pbs_options['branch_or_commit'] = os.environ['BRANCH_OR_COMMIT'] + base_pbs_options["conda_environment"] = os.environ["CONDA_ENVIRONMENT"] base_pbs_options['report_options'] = '' if debug: base_pbs_options['report_options'] += ' -d' diff --git a/testingScripts/runLocalTests.py b/testingScripts/runLocalTests.py new file mode 100755 index 00000000..985c2d46 --- /dev/null +++ b/testingScripts/runLocalTests.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python + +import argparse +import os +import subprocess +import pathlib +import datetime + +def create_command_line_parser(): + """Create the command-line argument parser. + + Create the parser for command-line arguments. + + Returns + ------- + parser : argparse.ArgumentParser + Command-line argument parser for this script. + + """ + parser = argparse.ArgumentParser(description="Script to help setup automated tests within a kaiju repo") + + parser.add_argument( + "-A", required=True, + help="Charge code to use when running tests." + ) + parser.add_argument( + "-ce", required=True, + help="Conda environment name to load with conda module" + ) + parser.add_argument( + "-p", default="economy", + help="Job priority to use when running tests (default: %(default)s)." + ) + + parser.add_argument( + "--unitTests", action='store_true',default=False, + help="Run unit tests (default: %(default)s)." + ) + parser.add_argument( + "--weeklyDash", action='store_true',default=False, + help="Run weekly dash (default: %(default)s)." + ) + parser.add_argument( + "--compTests", action='store_true',default=False, + help="Run default subset of comparative tests (default: %(default)s)." + ) + parser.add_argument( + "--compTestsFull", action='store_true',default=False, + help="Run full suite of comparative tests (over-rides --compTests) (default: %(default)s)." + ) + parser.add_argument( + "--buildTests", action='store_true',default=False, + help="Run build tests (default: %(default)s)." + ) + parser.add_argument( + "--icTests", action='store_true',default=False, + help="Run tests to build Initial Condition files (default: %(default)s)." + ) + parser.add_argument( + "--intelChecks", action='store_true',default=False, + help="Run Intel Inspector memory and thread check tests (default: %(default)s)." + ) + + parser.add_argument( + "--all", action='store_true',default=False, + help="Run all tests (default: %(default)s)." + ) + + return parser + +def main(): + """Helper script to run automated tests locally inside a kaiju repository + """ + # Set up the command-line parser. + parser = create_command_line_parser() + + # Parse the command-line arguments. + args = parser.parse_args() + + # find repo home directory + called_from = os.path.dirname(os.path.abspath(__file__)) + os.chdir(called_from) + os.chdir('..') + homeDir = os.getcwd() + + # Check for necessary environment variables + if 'KAIJUHOME' not in os.environ: + print("The setupEnvironment.sh script must be sourced for the repo this script resides in before calling it.") + exit() + if pathlib.Path(homeDir).resolve() != pathlib.Path(os.environ['KAIJUHOME']).resolve(): + print("The setupEnvironment.sh script must be sourced for the repo this script resides in before calling it.") + exit() + if 'KAIPYHOME' not in os.environ: + print("The setupEnvironment.sh script for ANY kaipy repo must be sourced before calling this.") + exit() + + # Set environment variables + os.environ['MAGE_TEST_ROOT'] = homeDir + os.environ['MAGE_TEST_RUNS_ROOT']=os.path.join(os.environ['MAGE_TEST_ROOT'],"test_runs") + os.environ['DERECHO_TESTING_ACCOUNT'] = args.A + os.environ['DERECHO_TESTING_QUEUE'] = 'main' + os.environ['DERECHO_TESTING_PRIORITY'] = args.p + os.environ['SLACK_BOT_TOKEN'] = '' # help ensure no accidental reporting to slack + os.environ['PYTHONUNBUFFERED']='TRUE' + os.environ['CONDA_ENVIRONMENT']=args.ce + gitBranch = subprocess.run(['git','branch','--show-current'], stdout=subprocess.PIPE).stdout.decode('utf-8') + if 'not a git repository' in gitBranch: + print("This script must be executed inside a kaiju git repository") + exit() + gitBranch = gitBranch.strip() + os.environ['BRANCH_OR_COMMIT'] = gitBranch + currenttime = datetime.datetime.now().strftime('%Y%m%d_%H%M%S') + test_set_dir = f"{currenttime}-{gitBranch}" + os.environ['MAGE_TEST_SET_ROOT'] = os.path.join(os.environ['MAGE_TEST_RUNS_ROOT'],test_set_dir) + os.makedirs(os.environ['MAGE_TEST_SET_ROOT'], exist_ok=True) + os.chdir(os.environ['MAGE_TEST_SET_ROOT']) + + print(f"Running tests on branch {gitBranch}") + print(f"Using charge code {args.A} with priority {args.p}") + print(f"Running in folder {test_set_dir}") + + # Adjust test options + if args.all: + args.unitTests = True + args.weeklyDash = True + args.compTests = True + args.compTestsFull = True + args.buildTests = True + args.icTests = True + args.intelChecks = True + + if args.compTestsFull: + args.compTests = False + + # Run Tests + if args.unitTests: + print("Running unit tests") + subprocess.call(['python', os.path.join(os.environ['MAGE_TEST_ROOT'],'testingScripts','unitTest.py'),'-tv']) + if args.weeklyDash: + print("Running weekly dash") + subprocess.call(['python', os.path.join(os.environ['MAGE_TEST_ROOT'],'testingScripts','weeklyDash.py'),'-tv']) + if args.compTests: + print("Running default comparative tests subset") + subprocess.call(['python', os.path.join(os.environ['MAGE_TEST_ROOT'],'testingScripts','relativeTests.py'),'-tv']) + if args.compTestsFull: + print("Running full comparative tests") + subprocess.call(['python', os.path.join(os.environ['MAGE_TEST_ROOT'],'testingScripts','relativeTests.py'),'-tva']) + if args.buildTests: + print("Running build tests") + subprocess.call(['python', os.path.join(os.environ['MAGE_TEST_ROOT'],'testingScripts','buildTest.py'),'-tv']) + if args.icTests: + print("Running Initial Condition tests") + subprocess.call(['python', os.path.join(os.environ['MAGE_TEST_ROOT'],'testingScripts','ICtest.py'),'-tv']) + if args.intelChecks: + print("Running memory and thread tests") + subprocess.call(['python', os.path.join(os.environ['MAGE_TEST_ROOT'],'testingScripts','intelChecks.py'),'-tv']) + +if __name__ == "__main__": + main() + diff --git a/testingScripts/unitTest.py b/testingScripts/unitTest.py index a9e59604..0d839b24 100644 --- a/testingScripts/unitTest.py +++ b/testingScripts/unitTest.py @@ -232,11 +232,15 @@ def unitTest(args: dict = None): if verbose: print("Copying compiled pFUnit binaries.") for directory in PFUNIT_BINARY_DIRECTORIES: - from_path = os.path.join(PFUNIT_HOME, directory) - to_path = os.path.join(KAIJU_EXTERNAL_DIRECTORY, directory) - if debug: - print(f"Copying {from_path} to {to_path}.") - shutil.copytree(from_path, to_path) + if not os.path.exists(os.path.join(KAIJU_EXTERNAL_DIRECTORY, directory)): + from_path = os.path.join(PFUNIT_HOME, directory) + to_path = os.path.join(KAIJU_EXTERNAL_DIRECTORY, directory) + if debug: + print(f"Copying {from_path} to {to_path}.") + shutil.copytree(from_path, to_path) + else: + if debug: + print(f"pFUnit directory {directory} already exists.") # ------------------------------------------------------------------------ @@ -414,13 +418,13 @@ def unitTest(args: dict = None): # Assemble common data to fill in the PBS templates. pbs_options = {} - pbs_options["account"] = os.environ["DERECHO_TESTING_ACCOUNT"] - pbs_options["queue"] = os.environ["DERECHO_TESTING_QUEUE"] - pbs_options["job_priority"] = os.environ["DERECHO_TESTING_PRIORITY"] - pbs_options["modules"] = module_names + pbs_options['account'] = os.environ['DERECHO_TESTING_ACCOUNT'] + pbs_options['queue'] = os.environ['DERECHO_TESTING_QUEUE'] + pbs_options['job_priority'] = os.environ['DERECHO_TESTING_PRIORITY'] + pbs_options['modules'] = module_names + pbs_options['kaijuhome'] = KAIJUHOME + pbs_options['branch_or_commit'] = BRANCH_OR_COMMIT pbs_options["conda_environment"] = os.environ["CONDA_ENVIRONMENT"] - pbs_options["kaijuhome"] = KAIJUHOME - pbs_options["branch_or_commit"] = BRANCH_OR_COMMIT # Go to the bin directory for testing. if verbose: @@ -430,14 +434,31 @@ def unitTest(args: dict = None): # -------------------------------------------------------------------- # Copy in inputs for unit test data generation. - for filename in UNIT_TEST_DATA_INPUT_FILES: - from_path = os.path.join( - UNIT_TEST_DATA_INPUT_DIRECTORY, filename - ) - to_path = os.path.join(".", filename) + if os.path.exists(UNIT_TEST_DATA_INPUT_DIRECTORY): + for filename in UNIT_TEST_DATA_INPUT_FILES: + from_path = os.path.join( + UNIT_TEST_DATA_INPUT_DIRECTORY, filename + ) + to_path = os.path.join('.', filename) + if debug: + print(f"Copying {from_path} to {to_path}.") + shutil.copyfile(from_path, to_path) + else: + cmd = "cda2wind.py -t0 2016-08-09T09:00:00 -t1 2016-08-09T11:00:00" if debug: - print(f"Copying {from_path} to {to_path}.") - shutil.copyfile(from_path, to_path) + print(f"cmd = {cmd}") + cproc = subprocess.run(cmd, shell=True, check=True, + text=True, capture_output=True) + cmd = "genLFM.py" + if debug: + print(f"cmd = {cmd}") + cproc = subprocess.run(cmd, shell=True, check=True, + text=True, capture_output=True) + cmd = "genRAIJU.py" + if debug: + print(f"cmd = {cmd}") + cproc = subprocess.run(cmd, shell=True, check=True, + text=True, capture_output=True) # Set options specific to the data generation job, then render the # template. @@ -511,8 +532,8 @@ def unitTest(args: dict = None): # Set options specific to the 1st non-case tests job, then render the # template. - pbs_options["job_name"] = "runNonCaseTests1" - pbs_options["walltime"] = "00:05:00" + pbs_options['job_name'] = 'runNonCaseTests1' + pbs_options['walltime'] = '01:00:00' if verbose: print(f"Creating {RUN_NON_CASE_TESTS_1_PBS_SCRIPT}.") pbs_content = run_non_case_tests_1_pbs_template.render(pbs_options) diff --git a/tests/unitTestReport-template.pbs b/tests/unitTestReport-template.pbs index c3de4dcd..6afc6fee 100644 --- a/tests/unitTestReport-template.pbs +++ b/tests/unitTestReport-template.pbs @@ -18,18 +18,28 @@ module load {{ module }} {%- endfor %} module list -echo 'Configuring python environment.' -mage_miniconda3="${HOME}/miniconda3" -mage_conda="${mage_miniconda3}/bin/conda" -__conda_setup="$($mage_conda 'shell.bash' 'hook' 2> /dev/null)" -if [ $? -eq 0 ]; then - eval "$__conda_setup" -else - if [ -f "$mage_miniconda3/etc/profile.d/conda.sh" ]; then - . "$mage_miniconda3/etc/profile.d/conda.sh" +echo 'Loading python environment.' +if [ -d "${mage_test_root}/miniconda3" ]; then + echo 'Loading local miniconda3' + mage_test_root='{{ mage_test_root }}' + export CONDARC="${mage_test_root}/.condarc" + export CONDA_ENVS_PATH="${mage_test_root}/.conda" + mage_miniconda3="${mage_test_root}/miniconda3" + mage_conda="${mage_miniconda3}/bin/conda" + __conda_setup="$($mage_conda 'shell.bash' 'hook' 2> /dev/null)" + if [ $? -eq 0 ]; then + eval "$__conda_setup" else - export PATH="$mage_miniconda3/bin:$PATH" + if [ -f "$mage_miniconda3/etc/profile.d/conda.sh" ]; then + . "$mage_miniconda3/etc/profile.d/conda.sh" + else + export PATH="$mage_miniconda3/bin:$PATH" + fi fi + unset __conda_setup +else + echo 'Loading conda module' + module load conda fi conda activate {{ conda_environment }}