#!/bin/bash

#PBS -N {{ sim_runid }}
#PBS -A {{ pbs_account }}
#PBS -q {{ pbs_queue }}
#PBS -l walltime={{ pbs_walltime }}
#PBS -l select={{ pbs_select }}:ncpus={{ pbs_ncpus }}:ompthreads={{ pbs_ompthreads }}
#PBS -m abe
#PBS -j oe

echo "Job $PBS_JOBID started at `date` on `hostname`."

# Specify the ID string for the run. This can be set to any desired string.
# PBS_JOBNAME is used here as an example, as it is set by the #PBS -N
# directive near the top of this file.
export RUNID=$PBS_JOBNAME

# Load the required modules for serial kaiju.
module purge
module load ncarenv/1.3
module load intel/19.1.1
module load ncarcompilers/0.5.0  # Must be after intel
module load hdf5/1.10.8
module load cmake/3.18.2

echo "The following modules are loaded:"
module list

# Define the kaiju installation location.
# NOTE: You MUST set this variable to the path to your kaiju directory, which
# is the top-level directory created when you cloned the kaiju repository.
export KAIJU_INSTALL_DIR={{ kaiju_home }}

# Set kaiju-related environment variables.
# This script sets KAIJUHOME and other environment variables.
source $KAIJU_INSTALL_DIR/scripts/setupEnvironment.sh

# Add the kaiju binary directory to the command path.
# NOTE: You should set this variable to the path to the bin subdirectory of
# your kaiju serial build directory.
export PATH={{ kaiju_build_bin }}:$PATH

# Set the OMP stack size to prevent a crash.
# If this setting is ignored, the model may cause the MPI kaiju code to crash
# with a segmentation fault and core dump. The value of "100M" was chosen
# ~arbitrarily; experimentation may allow a smaller value to be used.
export OMP_STACKSIZE=100M

echo "The active environment variables are:"
printenv

# Run the model. Direct output from the program is saved in a text file.
EXE=gamhelio.x
echo "Running $EXE on model $RUNID."
$EXE $RUNID.xml >& ${EXE}.${RUNID}.out

echo "Job $PBS_JOBID ended at `date` on `hostname`."
