#!/bin/bash

#PBS -N {{ job_name }}
#PBS -A {{ account }}
#PBS -q {{ queue }}
#PBS -l job_priority={{ job_priority }}
#PBS -l walltime={{ walltime }}
#PBS -l select=1:ncpus=128:mpiprocs=64:ompthreads=128
#PBS -j oe
#PBS -m abe

# Abort script on any error.
set -e

echo "Job $PBS_JOBID started at `date` on `hostname` in directory `pwd`."

echo 'Loading modules.'
module --force purge
{%- for module in modules %}
module load {{ module }}
{%- endfor %}
module list

echo 'Setting up MAGE environment.'
source {{ kaijuhome }}/scripts/setupEnvironment.sh

echo 'Setting environment variables.'
export OMP_NUM_THREADS=128
export MPI_TYPE_DEPTH=32
export KMP_STACKSIZE=128M
echo 'The active environment variables are:'
printenv

# Move to the directory containing the compiled code.
cd bin

echo 'Running GAMERA tests.'
date
./gamTests >& gamTests.out
date
echo 'GAMERA tests complete.'
echo | tail -n 3 ./gamTests.out

echo 'Running REMIX tests.'
date
./mixTests >& mixTests.out
date
echo 'REMIX tests complete.'
echo | tail -n 3 ./mixTests.out

echo 'Running SHELLGRID tests.'
date
./shgrTests >& shgrTests.out
date
echo 'SHELLGRID tests complete.'
echo | tail -n 3 ./shgrTests.out

echo 'Running VOLTRON tests.'
date
./voltTests >& voltTests.out
date
echo 'VOLTRON tests complete.'
echo | tail -n 3 ./voltTests.out

echo 'Running base MPI tests.'
MPICOMMAND="mpiexec $KAIJUHOME/scripts/preproc/pinCpuCores.sh"
date
${MPICOMMAND} ./baseMpiTests >& baseMpiTests.out
date
echo 'Base MPI tests complete.'
echo | tail -n 3 ./baseMpiTests.out

echo 'Running GAMERA MPI tests.'
date
${MPICOMMAND} ./gamMpiTests >& gamMpiTests.out
date
echo 'GAMERA MPI tests complete.'
echo | tail -n 3 ./gamMpiTests.out

echo "Job $PBS_JOBID ended at `date` on `hostname` in directory `pwd`."