#!/bin/bash

#PBS -N {{ sim_runid }}
#PBS -q {{ pbs_queue }}
#PBS -l walltime={{ pbs_walltime }}
#PBS -l select={{ pbs_select }}:ncpus={{ pbs_ncpus }}:ompthreads={{ pbs_ompthreads }}:model=bro
#PBS -m abe
#PBS -j oe

echo "Job $PBS_JOBID started at `date` on `hostname`."

# Specify the ID string for the run. This can be set to any desired string.
# PBS_JOBNAME is used here as an example, as it is set by the #PBS -N
# directive near the top of this file.
export RUNID=$PBS_JOBNAME

# Load the required modules for serial kaiju.
# NOTE: pleiades and electra do not have the "module restore" facility found
# on cheyenne. The modules must be explicitly loaded each time they are
# needed.
# NOTE: This set of modules assumes your kaiju installation was built using
# this same list of modules. If you used different modules at build time (for
# example, if you used a GNU compiler), update this list to use the modules
# from your build-time environment.

# Comment out the module lines for all systems except the one you are using.
# The default system is pleiades, using the Intel compiler.
module purge

# For pleiades/electra:
module load pkgsrc/2021Q2
module load comp-intel/2020.4.304
module load hdf5/1.8.18_serial

echo "The following modules are loaded:"
module list

echo "The following modules are loaded:"
module list

# Define the kaiju installation location.
# NOTE: You MUST set this variable to the path to your kaiju directory, which
# is the top-level directory created when you cloned the kaiju repository.
export KAIJU_INSTALL_DIR={{ kaiju_home }}

# Set kaiju-related environment variables.
# This script sets KAIJUHOME and other environment variables.
source $KAIJU_INSTALL_DIR/scripts/setupEnvironment.sh

# Add the kaiju binary directory to the command path.
# NOTE: You should set this variable to the path to the bin subdirectory of
# your kaiju serial build directory.
export PATH={{ kaiju_build_bin }}:$PATH

# Set the OMP stack size to prevent a crash.
# If this setting is ignored, the model may cause the MPI kaiju code to crash
# with a segmentation fault and core dump. The value of "100M" was chosen
# ~arbitrarily; experimentation may allow a smaller value to be used.
export OMP_STACKSIZE=100M

echo "The active environment variables are:"
printenv

# Run the model. Direct output from the program is saved in a text file.
EXE=gamhelio.x
echo "Running $EXE on model $RUNID."
# The omplace tool is used to ensure efficient pinning of MPI ranks and OMP
# threads to appropriate sockets and cores. If you omit omplace, your job
# will still run, but it probably be an order of magnitude slower than it
# would be when using omplace.
$EXE $RUNID.xml >& ${EXE}.${RUNID}.out

echo "Job $PBS_JOBID ended at `date` on `hostname`."
