#!/bin/bash

#PBS -N {{ job_name }}
#PBS -A {{ account }}
#PBS -q {{ queue }}
#PBS -l job_priority={{ job_priority }}
#PBS -l select=8:ncpus=128:mpiprocs=2:ompthreads=64+3:ncpus=128:mpiprocs=1:ompthreads=128
#PBS -l walltime={{ walltime }}
#PBS -j oe
#PBS -m abe

echo "Job $PBS_JOBID started at `date` on `hostname` in directory `pwd`."

echo 'Loading modules.'
module --force purge
{%- for module in modules %}
module load {{ module }}
{%- endfor %}
echo 'The currently loaded modules are:'
module list

echo 'Loading python environment.'
__conda_setup="$('/glade/u/home/ewinter/miniconda3/bin/conda' 'shell.bash' 'hook' 2> /dev/null)"
if [ $? -eq 0 ]; then
    eval "$__conda_setup"
else
    if [ -f "/glade/u/home/ewinter/miniconda3/etc/profile.d/conda.sh" ]; then
        . "/glade/u/home/ewinter/miniconda3/etc/profile.d/conda.sh"
    else
        export PATH="/glade/u/home/ewinter/miniconda3/bin:$PATH"
    fi
fi
unset __conda_setup

conda activate {{ conda_environment }}
echo "The current conda environment is ${CONDA_PREFIX}."

echo 'Setting up MAGE environment.'
source {{ kaijuhome }}/scripts/setupEnvironment.sh
echo "The kaiju software is located at ${KAIJUHOME}."

echo 'Setting environment variables.'
export TMPDIR={{ tmpdir }}
export SLACK_BOT_TOKEN={{ slack_bot_token }}
export MPI_TYPE_DEPTH=32
export KMP_STACKSIZE=128M
export MAGE_TEST_ROOT={{ mage_test_root }}
export MAGE_TEST_SET_ROOT={{ mage_test_set_root }}
export BRANCH_OR_COMMIT={{ branch_or_commit }}

echo 'The active environment variables are:'
printenv

# Build the code.
cmd="{{ cmake_cmd }}"
echo $cmd
eval $cmd
cmd="{{ make_cmd }}"
echo $cmd
eval $cmd

# Move to the bin directory.
cd bin

# Copy the run XML file.
cp ../weeklyDashGo.xml .

# Generate the LFM grid file.
echo 'Generating LFM grid file.'
{{ genLFM_cmd }}
echo "The LFM grid file is `ls lfm?.h5`."

# Generate the solar wind boundary condition file.
echo 'Generating the solar wind boundary condition file.'
{{ cda2wind_cmd }}
echo "The solar wind boundary condition file is `ls bcwind.h5`."

# Generate the RAIJU configuration file.
echo 'Generating the RAIJU configuration file.'
genRAIJU
echo "The RAIJU configuration file is `ls raijuconfig.h5`."

# Moved this setting here to avoid error from genLFM.py.
export OMP_NUM_THREADS=128

# Run the model.
MPICOMMAND="{{ mpiexec_cmd }}"
cmd="$MPICOMMAND {{ voltron_cmd }} >& weeklyDashGo.out"
echo "Model run command is:"
echo $cmd
eval $cmd

# Run the comparison. Post results to Slack if any test fails.
reference_run=`cat /glade/u/home/ewinter/mage_testing/test_runs/derecho_configuration_check_runs.txt`
cmd="python $KAIJUHOME/testingScripts/compare_mage_runs_numerical.py -sv --run_description='derecho configuration check' $reference_run `pwd`/weeklyDashGo.xml >& compare_mage_runs_numerical.out"
echo "Run comparison command is:"
echo $cmd
eval $cmd

echo "Job $PBS_JOBID ended at `date` on `hostname` in directory `pwd`."
